ant/src/org/sigrok/androidutils/ant/CopyLibsTask.java

   1 /*
   2  * This file is part of the sigrok-androidutils project.
   3  *
   4  * Copyright (C) 2014 Marcus Comstedt <marcus@mc.pp.se>
   5  *
   6  * This program is free software: you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation, either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 package org.sigrok.androidutils.ant;
  21
  22 import java.io.File;
  23 import java.io.IOException;
  24 import java.io.FileInputStream;
  25 import java.io.FileOutputStream;
  26 import java.io.InputStream;
  27 import java.io.OutputStream;
  28 import java.util.ArrayList;
  29 import java.util.Arrays;
  30 import java.util.Collections;
  31 import java.util.HashMap;
  32 import java.util.HashSet;
  33 import java.util.LinkedList;
  34 import java.util.Queue;
  35 import java.util.TreeSet;
  36 import java.util.Vector;
  37 import org.apache.tools.ant.BuildException;
  38 import org.apache.tools.ant.Task;
  39 import org.apache.tools.ant.types.FileSet;
  40 import org.apache.tools.ant.types.PatternSet;
  41 import org.apache.tools.ant.types.Resource;
  42 import org.apache.tools.ant.types.ResourceCollection;
  43 import org.apache.tools.ant.types.resources.FileProvider;
  44 import org.apache.tools.ant.types.selectors.SelectorUtils;
  45
  46 public class CopyLibsTask extends Task {
  47
  48
  49     private static BuildException buildException(Exception e)
  50     {
  51         if (e instanceof BuildException)
  52             return (BuildException)e;
  53         else
  54             return new BuildException(e);
  55     }
  56
  57     private static int indexOf(byte[] data, byte v, int start)
  58     {
  59         if (data != null)
  60             for (int i=start; i<data.length; i++)
  61                 if (data[i] == v)
  62                     return i;
  63         return -1;
  64     }
  65
  66     private static String fixSoname(String s)
  67     {
  68         int l = s.length();
  69         int i = s.lastIndexOf(".so");
  70         if (i >= 0 && i < l-3)
  71             return s.substring(0, i+3);
  72         else
  73             return s;
  74     }
  75
  76     protected class Library implements Comparable<Library> {
  77
  78         protected final File file;
  79         protected final ElfFile elf;
  80         protected final HashSet<String> needed;
  81         protected final Vector<String> rpath;
  82         protected final TreeSet<Range> fixups;
  83         protected final String subdir;
  84         protected String soname, destname;
  85         protected final HashSet<Library> dependencies;
  86         protected boolean dependedUpon;
  87
  88         protected class Range implements Comparable<Range> {
  89
  90             public final long start, end;
  91
  92             public int compareTo(Range r)
  93             {
  94                 if (start < r.start)
  95                     return -1;
  96                 else if (start > r.start)
  97                     return 1;
  98                 else if (end < r.end)
  99                     return -1;
 100                 else if (end > r.end)
 101                     return 1;
 102                 else
 103                     return 0;
 104             }
 105
 106             public Range(long start, long end)
 107             {
 108                 this.start = start;
 109                 this.end = end;
 110             }
 111         }
 112
 113         public int compareTo(Library l)
 114         {
 115             return destname.compareTo(l.destname);
 116         }
 117
 118         protected void addNeeded(String so)
 119         {
 120             needed.add(so);
 121         }
 122
 123         protected void addRpath(String rp)
 124         {
 125             for (String p : rp.split(":"))
 126                 if (!rpath.contains(p))
 127                     rpath.add(p);
 128         }
 129
 130         private String getDynstr(ElfFile.Dynamic d, byte[] s,
 131                                  long base) throws Exception
 132         {
 133             int offs = (int)d.d_val;
 134             int nul = indexOf(s, (byte)0, offs);
 135             if (nul < 0)
 136                 throw new Exception("Invalid dynamic string");
 137             String name = new String(s, offs, nul-offs, "US-ASCII");
 138             offs += base;
 139             if (d.d_tag == ElfFile.DT_RPATH) {
 140                 // Zap rpath
 141                 fixups.add(new Range(offs, offs+name.length()));
 142             } else {
 143                 String fix = fixSoname(name);
 144                 if (fix.length() < name.length()) {
 145                     fixups.add(new Range(offs+fix.length(),
 146                                          offs+name.length()));
 147                 }
 148             }
 149             return name;
 150         }
 151
 152         private void checkDynamic(ElfFile.SectionHeader dynsh,
 153                                   ElfFile.SectionHeader strsh) throws Exception
 154         {
 155             byte strs[] = new byte[(int)strsh.sh_size];
 156             elf.read(strsh, strs);
 157             for (ElfFile.Dynamic d : elf.readDynamic(dynsh))
 158                 if (d.d_tag == ElfFile.DT_NULL)
 159                     break;
 160                 else if (d.d_tag == ElfFile.DT_SONAME)
 161                     soname = getDynstr(d, strs, strsh.sh_offset);
 162                 else if (d.d_tag == ElfFile.DT_NEEDED)
 163                     addNeeded(getDynstr(d, strs, strsh.sh_offset));
 164                 else if (d.d_tag == ElfFile.DT_RPATH)
 165                     addRpath(getDynstr(d, strs, strsh.sh_offset));
 166         }
 167
 168         private void checkElf() throws Exception
 169         {
 170             for (ElfFile.SectionHeader sh : elf.secHeaders)
 171                 if (sh.sh_type == ElfFile.SHT_DYNAMIC)
 172                     checkDynamic(sh, elf.secHeaders[sh.sh_link]);
 173         }
 174
 175         protected File getDestName(File dest)
 176         {
 177             File d = new File(dest, subdir);
 178             File f = new File(d, destname);
 179             return f;
 180         }
 181
 182         protected void writeTo(File dest) throws IOException
 183         {
 184             FileInputStream is = new FileInputStream(file);
 185             FileOutputStream os = new FileOutputStream(dest);
 186             byte[] buf = new byte[65536];
 187             TreeSet<Range> ranges = new TreeSet<Range>(fixups);
 188             long offs = 0;
 189             outer: for(;;) {
 190                 long next = offs + buf.length;
 191                 if (!ranges.isEmpty()) {
 192                     next = ranges.first().start;
 193                 }
 194                 if (next > offs) {
 195                     long chunk = next-offs;
 196                     if (chunk > buf.length)
 197                         chunk = buf.length;
 198                     int r = is.read(buf, 0, (int)chunk);
 199                     if (r < 0)
 200                         break;
 201                     os.write(buf, 0, r);
 202                     offs += r;
 203                     continue;
 204                 }
 205                 while (!ranges.isEmpty() &&
 206                        ranges.first().start <= offs) {
 207                     Range rg = ranges.pollFirst();
 208                     if (rg.end > offs) {
 209                         long chunk = rg.end-offs;
 210                         while (chunk > 0) {
 211                             int slice = (chunk > buf.length?
 212                                          buf.length : (int)chunk);
 213                             int r = is.read(buf, 0, slice);
 214                             if (r < 0)
 215                                 break outer;
 216                             if (r > 0) {
 217                                 Arrays.fill(buf, 0, r, (byte)0);
 218                                 os.write(buf, 0, r);
 219                                 chunk -= r;
 220                             }
 221                         }
 222                         offs = rg.end;
 223                     }
 224                 }
 225             }
 226             os.close();
 227             is.close();
 228         }
 229
 230         protected Library(File f, String s) throws Exception
 231         {
 232             file = f;
 233             subdir = s;
 234             elf = new ElfFile(file);
 235             needed = new HashSet<String>();
 236             rpath = new Vector<String>();
 237             fixups = new TreeSet<Range>();
 238             soname = f.getName();
 239             dependencies = new HashSet<Library>();
 240             dependedUpon = false;
 241             checkElf();
 242             destname = fixSoname(soname);
 243         }
 244
 245         protected Library(Resource r) throws Exception
 246         {
 247             this(r.as(FileProvider.class).getFile(),
 248                  new File(r.getName()).getParent());
 249         }
 250
 251         public String toString() { return "Library("+file+")"; }
 252     };
 253
 254     protected class Worker
 255     {
 256         protected final int machine;
 257         protected final Queue<Library> workQueue;
 258         protected final HashMap<String,Library> knownLibs;
 259         protected final HashSet<Library> processedLibs;
 260         protected final HashSet<String> allDests;
 261         protected final Vector<String> rpath;
 262
 263         protected void addWork(Library l)
 264         {
 265             if (l == null)
 266                 return;
 267             Library kl = knownLibs.get(l.soname);
 268             if (kl == l)
 269                 return; // Already processed
 270             if (kl != null)
 271                 throw new BuildException("Multiple libs with the same soname "+
 272                                          l.soname);
 273             knownLibs.put(l.soname, l);
 274             if (allDests.contains(l.destname))
 275                 throw new BuildException("Multiple libs with simplified soname "+
 276                                          l.destname);
 277             allDests.add(l.destname);
 278             workQueue.add(l);
 279         }
 280
 281         protected void addRpath(Vector<String> rp)
 282         {
 283             for (String p : rp)
 284                 if (!rpath.contains(p))
 285                     rpath.add(p);
 286         }
 287
 288         protected void setDependency(Library l1, Library l2)
 289         {
 290             if (l2 == null) // Dependancy on external lib
 291                 return;
 292             l1.dependencies.add(l2);
 293             l2.dependedUpon = true;
 294         }
 295
 296         protected Library findLibInRpath(String s, String subdir)
 297             throws Exception
 298         {
 299             for (String p : rpath) {
 300                 File f = new File(p, s);
 301                 if (f.exists()) {
 302                     Library l = new Library(f, subdir);
 303                     if (l.elf.header.e_machine == machine)
 304                         return l;
 305                 }
 306             }
 307             return null;
 308         }
 309
 310         protected Library getLibForSoname(String s, String subdir)
 311             throws Exception
 312         {
 313             Library l = knownLibs.get(s);
 314             if (l != null)
 315                 return l;
 316             boolean include = false;
 317             for (String patt : patterns.getIncludePatterns(getProject()))
 318                 if (SelectorUtils.match(patt, s)) {
 319                     include = true;
 320                     break;
 321                 }
 322             if (!include) {
 323                 for (String patt : patterns.getExcludePatterns(getProject()))
 324                     if (SelectorUtils.match(patt, s))
 325                         return null;
 326             }
 327             l = findLibInRpath(s, subdir);
 328             if (l == null)
 329                 throw new Exception("Library "+s+" not found");
 330             addWork(l);
 331             return l;
 332         }
 333
 334         protected void process(Library l) throws Exception
 335         {
 336             if (processedLibs.contains(l))
 337                 return; // Already processed
 338             processedLibs.add(l);
 339             addRpath(l.rpath);
 340             for (String need : l.needed)
 341                 setDependency(l, getLibForSoname(need, l.subdir));
 342         }
 343
 344         protected Vector<Library> topoSort(HashSet<Library> libs)
 345         {
 346             Vector<Library> order = new Vector<Library>();
 347             for (Library chk : new HashSet<Library>(libs)) {
 348                 if (!chk.dependedUpon)
 349                     libs.remove(chk);
 350             }
 351             while (!libs.isEmpty()) {
 352                 HashSet<Library> leafs = new HashSet<Library>();
 353                 for (Library chk : new HashSet<Library>(libs)) {
 354                     if (chk.dependencies.isEmpty())
 355                         leafs.add(chk);
 356                 }
 357                 if (leafs.isEmpty())
 358                     throw new BuildException("Circular dependency found");
 359                 ArrayList<Library> llist = new ArrayList<Library>(leafs);
 360                 Collections.sort(llist);
 361                 order.addAll(llist);
 362                 libs.removeAll(leafs);
 363                 for (Library l : libs)
 364                     l.dependencies.removeAll(leafs);
 365             }
 366             return order;
 367         }
 368
 369         protected void execute() throws BuildException
 370         {
 371             try {
 372                 while (!workQueue.isEmpty())
 373                     process(workQueue.remove());
 374             } catch (Exception e) {
 375                 throw buildException(e);
 376             }
 377             if (property != null) {
 378                 Vector<Library> order =
 379                     topoSort(new HashSet<Library>(processedLibs));;
 380                 StringBuilder sb = new StringBuilder();
 381                 for (Library l : order) {
 382                     String name = l.destname;
 383                     if (name.startsWith("lib"))
 384                         name = name.substring(3);
 385                     if (name.endsWith(".so"))
 386                         name = name.substring(0, name.length()-3);
 387                     sb.append(" <item>");
 388                     sb.append(name);
 389                     sb.append("</item>\n");
 390                 }
 391                 String orderedLibs = sb.toString();
 392                 getProject().setNewProperty(property, orderedLibs);
 393             }
 394             for (Library chk : new HashSet<Library>(processedLibs)) {
 395                 File dest = chk.getDestName(destDir);
 396                 if (dest.exists() &&
 397                     dest.lastModified() >= chk.file.lastModified())
 398                     processedLibs.remove(chk);
 399                 dest = dest.getParentFile();
 400                 if (!dest.exists())
 401                     dest.mkdirs();
 402             }
 403             if (processedLibs.isEmpty())
 404                 return;
 405             log("Copying "+processedLibs.size()+" libraries into "+destDir);
 406             ArrayList<Library> libs = new ArrayList<Library>(processedLibs);
 407             Collections.sort(libs);
 408             try {
 409                 for (Library l : libs)
 410                     l.writeTo(l.getDestName(destDir));
 411             } catch (Exception e) {
 412                 throw buildException(e);
 413             }
 414         }
 415
 416         protected Worker(int mach)
 417         {
 418             machine = mach;
 419             workQueue = new LinkedList<Library>();
 420             knownLibs = new HashMap<String,Library>();
 421             processedLibs = new HashSet<Library>();
 422             allDests = new HashSet<String>();
 423             rpath = new Vector<String>();
 424         }
 425     };
 426
 427     protected File destDir = null;  // the destination directory
 428     protected Vector<ResourceCollection> rcs = new Vector<ResourceCollection>();
 429     protected PatternSet patterns = new PatternSet();
 430     protected String property = null;
 431
 432     public void setTodir(File destDir) {
 433         this.destDir = destDir;
 434     }
 435
 436     public void addFileset(FileSet set) {
 437         add(set);
 438     }
 439
 440     public void add(ResourceCollection res) {
 441         rcs.add(res);
 442     }
 443
 444     public PatternSet.NameEntry createExclude()
 445     {
 446         return patterns.createExclude();
 447     }
 448
 449     public PatternSet.NameEntry createInclude()
 450     {
 451         return patterns.createInclude();
 452     }
 453
 454     public void setProperty(String prop)
 455     {
 456         property = prop;
 457     }
 458
 459     public void execute() throws BuildException {
 460         HashMap<Integer,Worker> workers = new HashMap<Integer,Worker>();
 461         final int size = rcs.size();
 462         for (int i = 0; i < size; i++) {
 463             ResourceCollection rc = rcs.elementAt(i);
 464             for (Resource r : rc) {
 465                 if (!r.isExists()) {
 466                     String message = "Could not find library "
 467                         + r.toLongString() + " to copy.";
 468                     throw new BuildException(message);
 469                 }
 470                 Library l;
 471                 try {
 472                     l = new Library(r);
 473                 } catch (Exception e) {
 474                     throw buildException(e);
 475                 }
 476                 Integer m = new Integer(l.elf.header.e_machine);
 477                 Worker w = workers.get(m);
 478                 if (w == null)
 479                     workers.put(m, (w = new Worker(m.intValue())));
 480                 w.addWork(l);
 481             }
 482         }
 483         ArrayList<Integer> machines = new ArrayList<Integer>(workers.keySet());
 484         Collections.sort(machines);
 485         for (Integer m : machines)
 486             workers.get(m).execute();
 487     }
 488 }