ant/src/org/sigrok/androidutils/ant/CopyLibsTask.java

   1 /*
   2  * This file is part of the sigrok-androidutils project.
   3  *
   4  * Copyright (C) 2014 Marcus Comstedt <marcus@mc.pp.se>
   5  *
   6  * This program is free software: you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation, either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 package org.sigrok.androidutils.ant;
  21
  22 import java.io.File;
  23 import java.io.IOException;
  24 import java.io.FileInputStream;
  25 import java.io.FileOutputStream;
  26 import java.io.InputStream;
  27 import java.io.OutputStream;
  28 import java.util.ArrayList;
  29 import java.util.Arrays;
  30 import java.util.Collections;
  31 import java.util.HashMap;
  32 import java.util.HashSet;
  33 import java.util.LinkedList;
  34 import java.util.Queue;
  35 import java.util.TreeSet;
  36 import java.util.Vector;
  37 import org.apache.tools.ant.BuildException;
  38 import org.apache.tools.ant.DynamicAttribute;
  39 import org.apache.tools.ant.Task;
  40 import org.apache.tools.ant.types.FileSet;
  41 import org.apache.tools.ant.types.PatternSet;
  42 import org.apache.tools.ant.types.Resource;
  43 import org.apache.tools.ant.types.ResourceCollection;
  44 import org.apache.tools.ant.types.resources.FileProvider;
  45 import org.apache.tools.ant.types.selectors.SelectorUtils;
  46
  47 public class CopyLibsTask extends Task implements DynamicAttribute
  48 {
  49         private static final HashMap<String,String> blacklist;
  50
  51         static {
  52                 HashMap<String,String> bl = new HashMap<String,String>();
  53                 bl.put("libpcre.so", "libercp.so");
  54                 blacklist = bl;
  55         }
  56
  57         private static BuildException buildException(Exception e)
  58         {
  59                 if (e instanceof BuildException)
  60                         return (BuildException)e;
  61                 else
  62                         return new BuildException(e);
  63         }
  64
  65         private static int indexOf(byte[] data, byte v, int start)
  66         {
  67                 if (data != null) {
  68                         for (int i = start; i < data.length; i++) {
  69                                 if (data[i] == v)
  70                                         return i;
  71                         }
  72                 }
  73                 return -1;
  74         }
  75
  76         private static String fixSoname(String s)
  77         {
  78                 int l = s.length();
  79                 int i = s.lastIndexOf(".so");
  80
  81                 if (i >= 0 && i < (l - 3))
  82                         s = s.substring(0, i + 3);
  83
  84                 String bl = blacklist.get(s);
  85                 if (bl != null)
  86                         s = bl;
  87
  88                 return s;
  89         }
  90
  91         protected class Library implements Comparable<Library>
  92         {
  93                 protected final File file;
  94                 protected final ElfFile elf;
  95                 protected final HashSet<String> needed;
  96                 protected final Vector<String> rpath;
  97                 protected final TreeSet<Range> fixups;
  98                 protected final String subdir;
  99                 protected String soname, destname;
 100                 protected final HashSet<Library> dependencies;
 101                 protected boolean dependedUpon;
 102
 103                 protected class Range implements Comparable<Range>
 104                 {
 105                         public final long start, end;
 106                         public final byte[] replacement;
 107
 108                         public int compareTo(Range r)
 109                         {
 110                                 if (start < r.start)
 111                                         return -1;
 112                                 else if (start > r.start)
 113                                         return 1;
 114                                 else if (end < r.end)
 115                                         return -1;
 116                                 else if (end > r.end)
 117                                         return 1;
 118                                 else
 119                                         return 0;
 120                         }
 121
 122                         public Range(long start, long end, byte[] replacement)
 123                         {
 124                                 this.start = start;
 125                                 this.end = end;
 126                                 this.replacement = replacement;
 127                         }
 128
 129                         public Range(long start, long end)
 130                         {
 131                                 this(start, end, null);
 132                         }
 133                 }
 134
 135                 public int compareTo(Library l)
 136                 {
 137                         return destname.compareTo(l.destname);
 138                 }
 139
 140                 protected void addNeeded(String so)
 141                 {
 142                         needed.add(so);
 143                 }
 144
 145                 protected void addRpath(String rp)
 146                 {
 147                         for (String p : rp.split(":")) {
 148                                 if (!rpath.contains(p))
 149                                         rpath.add(p);
 150                         }
 151                 }
 152
 153                 private String getDynstr(ElfFile.Dynamic d, byte[] s,
 154                                          long base) throws Exception
 155                 {
 156                         int offs = (int)d.d_val;
 157                         int nul = indexOf(s, (byte)0, offs);
 158
 159                         if (nul < 0)
 160                                 throw new Exception("Invalid dynamic string");
 161
 162                         String name = new String(s, offs, nul-offs, "US-ASCII");
 163                         offs += base;
 164
 165                         if (d.d_tag == ElfFile.DT_RPATH ||
 166                             d.d_tag == ElfFile.DT_RUNPATH) {
 167                                 // Zap rpath,
 168                                 fixups.add(new Range(offs, offs + name.length()));
 169                         } else {
 170                                 String fix = fixSoname(name);
 171                                 if (fix.length() < name.length()) {
 172                                         fixups.add(new Range(offs + fix.length(),
 173                                                 offs + name.length()));
 174                                 }
 175                                 if (!fix.equals(name.substring(0, fix.length()))) {
 176                                         fixups.add(new Range(offs, offs + fix.length(), fix.getBytes("US-ASCII")));
 177                                 }
 178                         }
 179                         return name;
 180                 }
 181
 182                 private void checkDynamic(ElfFile.SectionHeader dynsh,
 183                                         ElfFile.SectionHeader strsh) throws Exception
 184                 {
 185                         byte strs[] = new byte[(int)strsh.sh_size];
 186
 187                         elf.read(strsh, strs);
 188
 189                         for (ElfFile.Dynamic d : elf.readDynamic(dynsh)) {
 190                                 if (d.d_tag == ElfFile.DT_NULL)
 191                                         break;
 192                                 else if (d.d_tag == ElfFile.DT_SONAME)
 193                                         soname = getDynstr(d, strs, strsh.sh_offset);
 194                                 else if (d.d_tag == ElfFile.DT_NEEDED)
 195                                         addNeeded(getDynstr(d, strs, strsh.sh_offset));
 196                                 else if (d.d_tag == ElfFile.DT_RPATH ||
 197                                          d.d_tag == ElfFile.DT_RUNPATH)
 198                                         addRpath(getDynstr(d, strs, strsh.sh_offset));
 199                         }
 200                 }
 201
 202                 private void checkElf() throws Exception
 203                 {
 204                         for (ElfFile.SectionHeader sh : elf.secHeaders) {
 205                                 if (sh.sh_type == ElfFile.SHT_DYNAMIC)
 206                                         checkDynamic(sh, elf.secHeaders[sh.sh_link]);
 207                         }
 208                 }
 209
 210                 protected File getDestName(File dest)
 211                 {
 212                         File d = (subdir == null? dest : new File(dest, subdir));
 213                         File f = new File(d, destname);
 214                         return f;
 215                 }
 216
 217                 protected void writeTo(File dest) throws IOException
 218                 {
 219                         FileInputStream is = new FileInputStream(file);
 220                         FileOutputStream os = new FileOutputStream(dest);
 221                         byte[] buf = new byte[65536];
 222                         TreeSet<Range> ranges = new TreeSet<Range>(fixups);
 223                         long offs = 0;
 224
 225                         outer: for(;;) {
 226                                 long next = offs + buf.length;
 227                                 if (!ranges.isEmpty())
 228                                         next = ranges.first().start;
 229                                 if (next > offs) {
 230                                         long chunk = next - offs;
 231                                         if (chunk > buf.length)
 232                                                 chunk = buf.length;
 233                                         int r = is.read(buf, 0, (int)chunk);
 234                                         if (r < 0)
 235                                                 break;
 236                                         os.write(buf, 0, r);
 237                                         offs += r;
 238                                         continue;
 239                                 }
 240                                 while (!ranges.isEmpty() && ranges.first().start <= offs) {
 241                                         Range rg = ranges.pollFirst();
 242                                         if (rg.end > offs) {
 243                                                 long chunk = rg.end - offs;
 244                                                 while (chunk > 0) {
 245                                                         int slice = (chunk > buf.length ? buf.length : (int)chunk);
 246                                                         int r = is.read(buf, 0, slice);
 247                                                         if (r < 0)
 248                                                                 break outer;
 249                                                         if (r > 0) {
 250                                                                 if (rg.replacement == null)
 251                                                                         Arrays.fill(buf, 0, r, (byte)0);
 252                                                                 else
 253                                                                         System.arraycopy(rg.replacement, (int)(offs-rg.start), buf, 0, r);
 254                                                                 os.write(buf, 0, r);
 255                                                                 chunk -= r;
 256                                                         }
 257                                                 }
 258                                                 offs = rg.end;
 259                                         }
 260                                 }
 261                         }
 262
 263                         os.close();
 264                         is.close();
 265                 }
 266
 267                 protected Library(File f, String s) throws Exception
 268                 {
 269                         file = f;
 270                         subdir = s;
 271                         elf = new ElfFile(file);
 272                         needed = new HashSet<String>();
 273                         rpath = new Vector<String>();
 274                         fixups = new TreeSet<Range>();
 275                         soname = f.getName();
 276                         dependencies = new HashSet<Library>();
 277                         dependedUpon = false;
 278                         checkElf();
 279                         destname = fixSoname(soname);
 280                 }
 281
 282                 protected Library(Resource r) throws Exception
 283                 {
 284                         this(r.as(FileProvider.class).getFile(),
 285                                 new File(r.getName()).getParent());
 286                 }
 287
 288                 public String toString()
 289                 {
 290                         return "Library(" + file + ")";
 291                 }
 292
 293         };
 294
 295         protected class Worker
 296         {
 297                 protected final int machine;
 298                 protected final Queue<Library> workQueue;
 299                 protected final HashMap<String,Library> knownLibs;
 300                 protected final HashSet<Library> processedLibs;
 301                 protected final HashSet<String> allDests;
 302                 protected final Vector<String> rpath;
 303
 304                 protected void addWork(Library l)
 305                 {
 306                         if (l == null)
 307                                 return;
 308                         Library kl = knownLibs.get(l.soname);
 309                         if (kl == l)
 310                                 return; // Already processed.
 311                         if (kl != null)
 312                                 throw new BuildException("Multiple libs with the same soname " + l.soname);
 313                         knownLibs.put(l.soname, l);
 314                         if (allDests.contains(l.destname))
 315                                 throw new BuildException("Multiple libs with simplified soname " + l.destname);
 316                         allDests.add(l.destname);
 317                         workQueue.add(l);
 318                 }
 319
 320                 protected void addRpath(Vector<String> rp)
 321                 {
 322                         for (String p : rp) {
 323                                 if (!rpath.contains(p))
 324                                         rpath.add(p);
 325                         }
 326                 }
 327
 328                 protected void setDependency(Library l1, Library l2)
 329                 {
 330                         if (l2 == null) // Dependency on external lib.
 331                                 return;
 332                         l1.dependencies.add(l2);
 333                         l2.dependedUpon = true;
 334                 }
 335
 336                 protected Library findLibInRpath(String s, String subdir)
 337                         throws Exception
 338                 {
 339                         for (String p : rpath) {
 340                                 File f = new File(p, s);
 341                                 if (f.exists()) {
 342                                         Library l = new Library(f, subdir);
 343                                         if (l.elf.header.e_machine == machine)
 344                                                 return l;
 345                                 }
 346                         }
 347                         return null;
 348                 }
 349
 350                 protected Library getLibForSoname(String s, String subdir)
 351                         throws Exception
 352                 {
 353                         Library l = knownLibs.get(s);
 354                         if (l != null)
 355                                 return l;
 356                         boolean include = false;
 357                         String[] includePatterns = patterns.getIncludePatterns(getProject());
 358                         if (includePatterns != null) {
 359                                 for (String patt : includePatterns) {
 360                                         if (SelectorUtils.match(patt, s)) {
 361                                                 include = true;
 362                                                 break;
 363                                         }
 364                                 }
 365                         }
 366                         if (!include) {
 367                                 String[] excludePatterns = patterns.getExcludePatterns(getProject());
 368                                 if (excludePatterns != null) {
 369                                         for (String patt : excludePatterns) {
 370                                                 if (SelectorUtils.match(patt, s))
 371                                                         return null;
 372                                         }
 373                                 }
 374                         }
 375                         l = findLibInRpath(s, subdir);
 376                         if (l == null)
 377                                 throw new Exception("Library " + s + " not found");
 378                         addWork(l);
 379                         return l;
 380                 }
 381
 382                 protected void process(Library l) throws Exception
 383                 {
 384                         if (processedLibs.contains(l))
 385                                 return; // Already processed.
 386                         processedLibs.add(l);
 387                         addRpath(l.rpath);
 388                         for (String need : l.needed)
 389                                 setDependency(l, getLibForSoname(need, l.subdir));
 390                 }
 391
 392                 protected Vector<Library> topoSort(HashSet<Library> libs)
 393                 {
 394                         Vector<Library> order = new Vector<Library>();
 395                         for (Library chk : new HashSet<Library>(libs)) {
 396                                 if (!chk.dependedUpon)
 397                                         libs.remove(chk);
 398                         }
 399                         while (!libs.isEmpty()) {
 400                                 HashSet<Library> leafs = new HashSet<Library>();
 401                                 for (Library chk : new HashSet<Library>(libs)) {
 402                                         if (chk.dependencies.isEmpty())
 403                                                 leafs.add(chk);
 404                                 }
 405                                 if (leafs.isEmpty())
 406                                         throw new BuildException("Circular dependency found");
 407                                 ArrayList<Library> llist = new ArrayList<Library>(leafs);
 408                                 Collections.sort(llist);
 409                                 order.addAll(llist);
 410                                 libs.removeAll(leafs);
 411                                 for (Library l : libs)
 412                                         l.dependencies.removeAll(leafs);
 413                         }
 414                         return order;
 415                 }
 416
 417                 protected void execute() throws BuildException
 418                 {
 419                         try {
 420                                 while (!workQueue.isEmpty())
 421                                         process(workQueue.remove());
 422                         } catch (Exception e) {
 423                                 throw buildException(e);
 424                         }
 425                         if (property != null) {
 426                                 Vector<Library> order =
 427                                         topoSort(new HashSet<Library>(processedLibs));
 428                                 StringBuilder sb = new StringBuilder();
 429                                 for (Library l : order) {
 430                                         String name = l.destname;
 431                                         if (name.startsWith("lib"))
 432                                                 name = name.substring(3);
 433                                         if (name.endsWith(".so"))
 434                                                 name = name.substring(0, name.length() - 3);
 435                                         sb.append("     <item>");
 436                                         sb.append(name);
 437                                         sb.append("</item>\n");
 438                                 }
 439                                 String orderedLibs = sb.toString();
 440                                 getProject().setNewProperty(property, orderedLibs);
 441                         }
 442                         for (Library chk : new HashSet<Library>(processedLibs)) {
 443                                 File dest = chk.getDestName(destDir);
 444                                 if (dest.exists() &&
 445                                         dest.lastModified() >= chk.file.lastModified())
 446                                         processedLibs.remove(chk);
 447                                 dest = dest.getParentFile();
 448                                 if (!dest.exists())
 449                                         dest.mkdirs();
 450                         }
 451                         if (processedLibs.isEmpty())
 452                                 return;
 453                         log("Copying " + processedLibs.size() + " libraries into " + destDir);
 454                         ArrayList<Library> libs = new ArrayList<Library>(processedLibs);
 455                         Collections.sort(libs);
 456                         try {
 457                                 for (Library l : libs)
 458                                         l.writeTo(l.getDestName(destDir));
 459                         } catch (Exception e) {
 460                                 throw buildException(e);
 461                         }
 462                 }
 463
 464                 protected Worker(int mach)
 465                 {
 466                         machine = mach;
 467                         workQueue = new LinkedList<Library>();
 468                         knownLibs = new HashMap<String,Library>();
 469                         processedLibs = new HashSet<Library>();
 470                         allDests = new HashSet<String>();
 471                         rpath = new Vector<String>();
 472                 }
 473
 474         };
 475
 476         protected File destDir = null; // The destination directory.
 477         protected Vector<ResourceCollection> rcs = new Vector<ResourceCollection>();
 478         protected PatternSet patterns = new PatternSet();
 479         protected String property = null;
 480         protected Vector<String> rpath = new Vector<String>();
 481
 482         public void setTodir(File destDir)
 483         {
 484                 this.destDir = destDir;
 485         }
 486
 487         public void setDynamicAttribute(String name, String value)
 488         {
 489                 if ("rpath-link".equals(name))
 490                         this.rpath.add(value);
 491                 else
 492                         throw new BuildException("copylibs doesn't support the \"" + name + "\" attribute");
 493         }
 494
 495         public void addFileset(FileSet set)
 496         {
 497                 add(set);
 498         }
 499
 500         public void add(ResourceCollection res)
 501         {
 502                 rcs.add(res);
 503         }
 504
 505         public PatternSet.NameEntry createExclude()
 506         {
 507                 return patterns.createExclude();
 508         }
 509
 510         public PatternSet.NameEntry createInclude()
 511         {
 512                 return patterns.createInclude();
 513         }
 514
 515         public void setProperty(String prop)
 516         {
 517                 property = prop;
 518         }
 519
 520         public void execute() throws BuildException
 521         {
 522                 HashMap<Integer,Worker> workers = new HashMap<Integer,Worker>();
 523                 final int size = rcs.size();
 524
 525                 for (int i = 0; i < size; i++) {
 526                         ResourceCollection rc = rcs.elementAt(i);
 527                         for (Resource r : rc) {
 528                                 if (!r.isExists()) {
 529                                         String message = "Could not find library "
 530                                                 + r.toLongString() + " to copy.";
 531                                         throw new BuildException(message);
 532                                 }
 533                                 Library l;
 534                                 try {
 535                                         l = new Library(r);
 536                                 } catch (Exception e) {
 537                                         throw buildException(e);
 538                                 }
 539                                 Integer m = new Integer(l.elf.header.e_machine);
 540                                 Worker w = workers.get(m);
 541                                 if (w == null) {
 542                                         workers.put(m, (w = new Worker(m.intValue())));
 543                                         w.addRpath(rpath);
 544                                 }
 545                                 w.addWork(l);
 546                         }
 547                 }
 548                 ArrayList<Integer> machines = new ArrayList<Integer>(workers.keySet());
 549                 Collections.sort(machines);
 550                 for (Integer m : machines)
 551                         workers.get(m).execute();
 552         }
 553 }