ant/src/org/sigrok/androidutils/ant/CopyLibsTask.java

   1 /*
   2  * This file is part of the sigrok-androidutils project.
   3  *
   4  * Copyright (C) 2014 Marcus Comstedt <marcus@mc.pp.se>
   5  *
   6  * This program is free software: you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation, either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 package org.sigrok.androidutils.ant;
  21
  22 import java.io.File;
  23 import java.io.IOException;
  24 import java.io.FileInputStream;
  25 import java.io.FileOutputStream;
  26 import java.io.InputStream;
  27 import java.io.OutputStream;
  28 import java.util.ArrayList;
  29 import java.util.Arrays;
  30 import java.util.Collections;
  31 import java.util.HashMap;
  32 import java.util.HashSet;
  33 import java.util.LinkedList;
  34 import java.util.Queue;
  35 import java.util.TreeSet;
  36 import java.util.Vector;
  37 import org.apache.tools.ant.BuildException;
  38 import org.apache.tools.ant.Task;
  39 import org.apache.tools.ant.types.FileSet;
  40 import org.apache.tools.ant.types.PatternSet;
  41 import org.apache.tools.ant.types.Resource;
  42 import org.apache.tools.ant.types.ResourceCollection;
  43 import org.apache.tools.ant.types.resources.FileProvider;
  44 import org.apache.tools.ant.types.selectors.SelectorUtils;
  45
  46 public class CopyLibsTask extends Task
  47 {
  48         private static BuildException buildException(Exception e)
  49         {
  50                 if (e instanceof BuildException)
  51                         return (BuildException)e;
  52                 else
  53                         return new BuildException(e);
  54         }
  55
  56         private static int indexOf(byte[] data, byte v, int start)
  57         {
  58                 if (data != null) {
  59                         for (int i = start; i < data.length; i++) {
  60                                 if (data[i] == v)
  61                                         return i;
  62                         }
  63                 }
  64                 return -1;
  65         }
  66
  67         private static String fixSoname(String s)
  68         {
  69                 int l = s.length();
  70                 int i = s.lastIndexOf(".so");
  71
  72                 if (i >= 0 && i < (l - 3))
  73                         return s.substring(0, i + 3);
  74                 else
  75                         return s;
  76         }
  77
  78         protected class Library implements Comparable<Library>
  79         {
  80                 protected final File file;
  81                 protected final ElfFile elf;
  82                 protected final HashSet<String> needed;
  83                 protected final Vector<String> rpath;
  84                 protected final TreeSet<Range> fixups;
  85                 protected final String subdir;
  86                 protected String soname, destname;
  87                 protected final HashSet<Library> dependencies;
  88                 protected boolean dependedUpon;
  89
  90                 protected class Range implements Comparable<Range>
  91                 {
  92                         public final long start, end;
  93
  94                         public int compareTo(Range r)
  95                         {
  96                                 if (start < r.start)
  97                                         return -1;
  98                                 else if (start > r.start)
  99                                         return 1;
 100                                 else if (end < r.end)
 101                                         return -1;
 102                                 else if (end > r.end)
 103                                         return 1;
 104                                 else
 105                                         return 0;
 106                         }
 107
 108                         public Range(long start, long end)
 109                         {
 110                                 this.start = start;
 111                                 this.end = end;
 112                         }
 113                 }
 114
 115                 public int compareTo(Library l)
 116                 {
 117                         return destname.compareTo(l.destname);
 118                 }
 119
 120                 protected void addNeeded(String so)
 121                 {
 122                         needed.add(so);
 123                 }
 124
 125                 protected void addRpath(String rp)
 126                 {
 127                         for (String p : rp.split(":")) {
 128                                 if (!rpath.contains(p))
 129                                         rpath.add(p);
 130                         }
 131                 }
 132
 133                 private String getDynstr(ElfFile.Dynamic d, byte[] s,
 134                                          long base) throws Exception
 135                 {
 136                         int offs = (int)d.d_val;
 137                         int nul = indexOf(s, (byte)0, offs);
 138
 139                         if (nul < 0)
 140                                 throw new Exception("Invalid dynamic string");
 141
 142                         String name = new String(s, offs, nul-offs, "US-ASCII");
 143                         offs += base;
 144
 145                         if (d.d_tag == ElfFile.DT_RPATH) {
 146                                 // Zap rpath,
 147                                 fixups.add(new Range(offs, offs + name.length()));
 148                         } else {
 149                                 String fix = fixSoname(name);
 150                                 if (fix.length() < name.length()) {
 151                                         fixups.add(new Range(offs + fix.length(),
 152                                                 offs + name.length()));
 153                                 }
 154                         }
 155                         return name;
 156                 }
 157
 158                 private void checkDynamic(ElfFile.SectionHeader dynsh,
 159                                         ElfFile.SectionHeader strsh) throws Exception
 160                 {
 161                         byte strs[] = new byte[(int)strsh.sh_size];
 162
 163                         elf.read(strsh, strs);
 164
 165                         for (ElfFile.Dynamic d : elf.readDynamic(dynsh)) {
 166                                 if (d.d_tag == ElfFile.DT_NULL)
 167                                         break;
 168                                 else if (d.d_tag == ElfFile.DT_SONAME)
 169                                         soname = getDynstr(d, strs, strsh.sh_offset);
 170                                 else if (d.d_tag == ElfFile.DT_NEEDED)
 171                                         addNeeded(getDynstr(d, strs, strsh.sh_offset));
 172                                 else if (d.d_tag == ElfFile.DT_RPATH)
 173                                         addRpath(getDynstr(d, strs, strsh.sh_offset));
 174                         }
 175                 }
 176
 177                 private void checkElf() throws Exception
 178                 {
 179                         for (ElfFile.SectionHeader sh : elf.secHeaders) {
 180                                 if (sh.sh_type == ElfFile.SHT_DYNAMIC)
 181                                         checkDynamic(sh, elf.secHeaders[sh.sh_link]);
 182                         }
 183                 }
 184
 185                 protected File getDestName(File dest)
 186                 {
 187                         File d = new File(dest, subdir);
 188                         File f = new File(d, destname);
 189                         return f;
 190                 }
 191
 192                 protected void writeTo(File dest) throws IOException
 193                 {
 194                         FileInputStream is = new FileInputStream(file);
 195                         FileOutputStream os = new FileOutputStream(dest);
 196                         byte[] buf = new byte[65536];
 197                         TreeSet<Range> ranges = new TreeSet<Range>(fixups);
 198                         long offs = 0;
 199
 200                         outer: for(;;) {
 201                                 long next = offs + buf.length;
 202                                 if (!ranges.isEmpty())
 203                                         next = ranges.first().start;
 204                                 if (next > offs) {
 205                                         long chunk = next - offs;
 206                                         if (chunk > buf.length)
 207                                                 chunk = buf.length;
 208                                         int r = is.read(buf, 0, (int)chunk);
 209                                         if (r < 0)
 210                                                 break;
 211                                         os.write(buf, 0, r);
 212                                         offs += r;
 213                                         continue;
 214                                 }
 215                                 while (!ranges.isEmpty() && ranges.first().start <= offs) {
 216                                         Range rg = ranges.pollFirst();
 217                                         if (rg.end > offs) {
 218                                                 long chunk = rg.end - offs;
 219                                                 while (chunk > 0) {
 220                                                         int slice = (chunk > buf.length ? buf.length : (int)chunk);
 221                                                         int r = is.read(buf, 0, slice);
 222                                                         if (r < 0)
 223                                                                 break outer;
 224                                                         if (r > 0) {
 225                                                                 Arrays.fill(buf, 0, r, (byte)0);
 226                                                                 os.write(buf, 0, r);
 227                                                                 chunk -= r;
 228                                                         }
 229                                                 }
 230                                                 offs = rg.end;
 231                                         }
 232                                 }
 233                         }
 234
 235                         os.close();
 236                         is.close();
 237                 }
 238
 239                 protected Library(File f, String s) throws Exception
 240                 {
 241                         file = f;
 242                         subdir = s;
 243                         elf = new ElfFile(file);
 244                         needed = new HashSet<String>();
 245                         rpath = new Vector<String>();
 246                         fixups = new TreeSet<Range>();
 247                         soname = f.getName();
 248                         dependencies = new HashSet<Library>();
 249                         dependedUpon = false;
 250                         checkElf();
 251                         destname = fixSoname(soname);
 252                 }
 253
 254                 protected Library(Resource r) throws Exception
 255                 {
 256                         this(r.as(FileProvider.class).getFile(),
 257                                 new File(r.getName()).getParent());
 258                 }
 259
 260                 public String toString()
 261                 {
 262                         return "Library(" + file + ")";
 263                 }
 264
 265         };
 266
 267         protected class Worker
 268         {
 269                 protected final int machine;
 270                 protected final Queue<Library> workQueue;
 271                 protected final HashMap<String,Library> knownLibs;
 272                 protected final HashSet<Library> processedLibs;
 273                 protected final HashSet<String> allDests;
 274                 protected final Vector<String> rpath;
 275
 276                 protected void addWork(Library l)
 277                 {
 278                         if (l == null)
 279                                 return;
 280                         Library kl = knownLibs.get(l.soname);
 281                         if (kl == l)
 282                                 return; // Already processed.
 283                         if (kl != null)
 284                                 throw new BuildException("Multiple libs with the same soname " + l.soname);
 285                         knownLibs.put(l.soname, l);
 286                         if (allDests.contains(l.destname))
 287                                 throw new BuildException("Multiple libs with simplified soname " + l.destname);
 288                         allDests.add(l.destname);
 289                         workQueue.add(l);
 290                 }
 291
 292                 protected void addRpath(Vector<String> rp)
 293                 {
 294                         for (String p : rp) {
 295                                 if (!rpath.contains(p))
 296                                         rpath.add(p);
 297                         }
 298                 }
 299
 300                 protected void setDependency(Library l1, Library l2)
 301                 {
 302                         if (l2 == null) // Dependancy on external lib.
 303                                 return;
 304                         l1.dependencies.add(l2);
 305                         l2.dependedUpon = true;
 306                 }
 307
 308                 protected Library findLibInRpath(String s, String subdir)
 309                         throws Exception
 310                 {
 311                         for (String p : rpath) {
 312                                 File f = new File(p, s);
 313                                 if (f.exists()) {
 314                                         Library l = new Library(f, subdir);
 315                                         if (l.elf.header.e_machine == machine)
 316                                                 return l;
 317                                 }
 318                         }
 319                         return null;
 320                 }
 321
 322                 protected Library getLibForSoname(String s, String subdir)
 323                         throws Exception
 324                 {
 325                         Library l = knownLibs.get(s);
 326                         if (l != null)
 327                                 return l;
 328                         boolean include = false;
 329                         for (String patt : patterns.getIncludePatterns(getProject())) {
 330                                 if (SelectorUtils.match(patt, s)) {
 331                                         include = true;
 332                                         break;
 333                                 }
 334                         }
 335                         if (!include) {
 336                                 for (String patt : patterns.getExcludePatterns(getProject())) {
 337                                         if (SelectorUtils.match(patt, s))
 338                                                 return null;
 339                                 }
 340                         }
 341                         l = findLibInRpath(s, subdir);
 342                         if (l == null)
 343                                 throw new Exception("Library " + s + " not found");
 344                         addWork(l);
 345                         return l;
 346                 }
 347
 348                 protected void process(Library l) throws Exception
 349                 {
 350                         if (processedLibs.contains(l))
 351                                 return; // Already processed
 352                         processedLibs.add(l);
 353                         addRpath(l.rpath);
 354                         for (String need : l.needed)
 355                                 setDependency(l, getLibForSoname(need, l.subdir));
 356                 }
 357
 358                 protected Vector<Library> topoSort(HashSet<Library> libs)
 359                 {
 360                         Vector<Library> order = new Vector<Library>();
 361                         for (Library chk : new HashSet<Library>(libs)) {
 362                                 if (!chk.dependedUpon)
 363                                         libs.remove(chk);
 364                         }
 365                         while (!libs.isEmpty()) {
 366                                 HashSet<Library> leafs = new HashSet<Library>();
 367                                 for (Library chk : new HashSet<Library>(libs)) {
 368                                         if (chk.dependencies.isEmpty())
 369                                                 leafs.add(chk);
 370                                 }
 371                                 if (leafs.isEmpty())
 372                                         throw new BuildException("Circular dependency found");
 373                                 ArrayList<Library> llist = new ArrayList<Library>(leafs);
 374                                 Collections.sort(llist);
 375                                 order.addAll(llist);
 376                                 libs.removeAll(leafs);
 377                                 for (Library l : libs)
 378                                         l.dependencies.removeAll(leafs);
 379                         }
 380                         return order;
 381                 }
 382
 383                 protected void execute() throws BuildException
 384                 {
 385                         try {
 386                                 while (!workQueue.isEmpty())
 387                                         process(workQueue.remove());
 388                         } catch (Exception e) {
 389                                 throw buildException(e);
 390                         }
 391                         if (property != null) {
 392                                 Vector<Library> order =
 393                                         topoSort(new HashSet<Library>(processedLibs));;
 394                                 StringBuilder sb = new StringBuilder();
 395                                 for (Library l : order) {
 396                                         String name = l.destname;
 397                                         if (name.startsWith("lib"))
 398                                                 name = name.substring(3);
 399                                         if (name.endsWith(".so"))
 400                                                 name = name.substring(0, name.length() - 3);
 401                                         sb.append("     <item>");
 402                                         sb.append(name);
 403                                         sb.append("</item>\n");
 404                                 }
 405                                 String orderedLibs = sb.toString();
 406                                 getProject().setNewProperty(property, orderedLibs);
 407                         }
 408                         for (Library chk : new HashSet<Library>(processedLibs)) {
 409                                 File dest = chk.getDestName(destDir);
 410                                 if (dest.exists() &&
 411                                         dest.lastModified() >= chk.file.lastModified())
 412                                         processedLibs.remove(chk);
 413                                 dest = dest.getParentFile();
 414                                 if (!dest.exists())
 415                                         dest.mkdirs();
 416                         }
 417                         if (processedLibs.isEmpty())
 418                                 return;
 419                         log("Copying " + processedLibs.size() + " libraries into " + destDir);
 420                         ArrayList<Library> libs = new ArrayList<Library>(processedLibs);
 421                         Collections.sort(libs);
 422                         try {
 423                                 for (Library l : libs)
 424                                         l.writeTo(l.getDestName(destDir));
 425                         } catch (Exception e) {
 426                                 throw buildException(e);
 427                         }
 428                 }
 429
 430                 protected Worker(int mach)
 431                 {
 432                         machine = mach;
 433                         workQueue = new LinkedList<Library>();
 434                         knownLibs = new HashMap<String,Library>();
 435                         processedLibs = new HashSet<Library>();
 436                         allDests = new HashSet<String>();
 437                         rpath = new Vector<String>();
 438                 }
 439
 440         };
 441
 442         protected File destDir = null;  // The destination directory.
 443         protected Vector<ResourceCollection> rcs = new Vector<ResourceCollection>();
 444         protected PatternSet patterns = new PatternSet();
 445         protected String property = null;
 446
 447         public void setTodir(File destDir)
 448         {
 449                 this.destDir = destDir;
 450         }
 451
 452         public void addFileset(FileSet set)
 453         {
 454                 add(set);
 455         }
 456
 457         public void add(ResourceCollection res)
 458         {
 459                 rcs.add(res);
 460         }
 461
 462         public PatternSet.NameEntry createExclude()
 463         {
 464                 return patterns.createExclude();
 465         }
 466
 467         public PatternSet.NameEntry createInclude()
 468         {
 469                 return patterns.createInclude();
 470         }
 471
 472         public void setProperty(String prop)
 473         {
 474                 property = prop;
 475         }
 476
 477         public void execute() throws BuildException
 478         {
 479                 HashMap<Integer,Worker> workers = new HashMap<Integer,Worker>();
 480                 final int size = rcs.size();
 481
 482                 for (int i = 0; i < size; i++) {
 483                         ResourceCollection rc = rcs.elementAt(i);
 484                         for (Resource r : rc) {
 485                                 if (!r.isExists()) {
 486                                         String message = "Could not find library "
 487                                                 + r.toLongString() + " to copy.";
 488                                         throw new BuildException(message);
 489                                 }
 490                                 Library l;
 491                                 try {
 492                                         l = new Library(r);
 493                                 } catch (Exception e) {
 494                                         throw buildException(e);
 495                                 }
 496                                 Integer m = new Integer(l.elf.header.e_machine);
 497                                 Worker w = workers.get(m);
 498                                 if (w == null)
 499                                         workers.put(m, (w = new Worker(m.intValue())));
 500                                 w.addWork(l);
 501                         }
 502                 }
 503                 ArrayList<Integer> machines = new ArrayList<Integer>(workers.keySet());
 504                 Collections.sort(machines);
 505                 for (Integer m : machines)
 506                         workers.get(m).execute();
 507         }
 508 }