Merge "Bump dx version to 1.11" into lmp-mr1-dev
diff --git a/dx/src/com/android/dx/cf/direct/ClassPathOpener.java b/dx/src/com/android/dx/cf/direct/ClassPathOpener.java
index c9fe275..26fbca0 100644
--- a/dx/src/com/android/dx/cf/direct/ClassPathOpener.java
+++ b/dx/src/com/android/dx/cf/direct/ClassPathOpener.java
@@ -242,9 +242,6 @@
*/
private boolean processArchive(File file) throws IOException {
ZipFile zip = new ZipFile(file);
- ByteArrayOutputStream baos = new ByteArrayOutputStream(40000);
- byte[] buf = new byte[20000];
- boolean any = false;
ArrayList<? extends java.util.zip.ZipEntry> entriesList
= Collections.list(zip.entries());
@@ -259,28 +256,31 @@
consumer.onProcessArchiveStart(file);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream(40000);
+ byte[] buf = new byte[20000];
+ boolean any = false;
+
for (ZipEntry one : entriesList) {
- if (one.isDirectory()) {
- continue;
- }
+ final boolean isDirectory = one.isDirectory();
String path = one.getName();
if (filter.accept(path)) {
- InputStream in = zip.getInputStream(one);
+ final byte[] bytes;
+ if (!isDirectory) {
+ InputStream in = zip.getInputStream(one);
- baos.reset();
- for (;;) {
- int amt = in.read(buf);
- if (amt < 0) {
- break;
+ baos.reset();
+ int read;
+ while ((read = in.read(buf)) != -1) {
+ baos.write(buf, 0, read);
}
- baos.write(buf, 0, amt);
+ in.close();
+ bytes = baos.toByteArray();
+ } else {
+ bytes = new byte[0];
}
- in.close();
-
- byte[] bytes = baos.toByteArray();
any |= consumer.processFileBytes(path, one.getTime(), bytes);
}
}
diff --git a/dx/src/com/android/dx/command/DxConsole.java b/dx/src/com/android/dx/command/DxConsole.java
index 9ce9836..919de8c 100644
--- a/dx/src/com/android/dx/command/DxConsole.java
+++ b/dx/src/com/android/dx/command/DxConsole.java
@@ -16,6 +16,8 @@
package com.android.dx.command;
+import java.io.IOException;
+import java.io.OutputStream;
import java.io.PrintStream;
/**
@@ -34,4 +36,15 @@
* Error output stream. Links to {@code System.err} by default.
*/
public static PrintStream err = System.err;
+
+ /**
+ * Output stream which prints to nowhere.
+ */
+ public static final PrintStream noop = new PrintStream(new OutputStream() {
+
+ @Override
+ public void write(int b) throws IOException {
+ // noop
+ }
+ });
}
diff --git a/dx/src/com/android/dx/command/Main.java b/dx/src/com/android/dx/command/Main.java
index 9834987..a0be5bd 100644
--- a/dx/src/com/android/dx/command/Main.java
+++ b/dx/src/com/android/dx/command/Main.java
@@ -33,7 +33,7 @@
"[--dump-width=<n>]\n" +
" [--dump-method=<name>[*]] [--verbose-dump] [--no-files] " +
"[--core-library]\n" +
- " [--num-threads=<n>] [--incremental] [--force-jumbo]\n" +
+ " [--num-threads=<n>] [--incremental] [--force-jumbo] [--no-warning]\n" +
" [--multi-dex [--main-dex-list=<file> [--minimal-main-dex]]\n" +
" [--input-list=<file>]\n" +
" [<file>.class | <file>.{zip,jar,apk} | <directory>] ...\n" +
diff --git a/dx/src/com/android/dx/command/dexer/Main.java b/dx/src/com/android/dx/command/dexer/Main.java
index 4a3d195..b9c6ec9 100644
--- a/dx/src/com/android/dx/command/dexer/Main.java
+++ b/dx/src/com/android/dx/command/dexer/Main.java
@@ -64,12 +64,14 @@
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
+import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.jar.Attributes;
@@ -179,11 +181,39 @@
/** Library .dex files to merge into the output .dex. */
private static final List<byte[]> libraryDexBuffers = new ArrayList<byte[]>();
- /** thread pool object used for multi-threaded file processing */
- private static ExecutorService threadPool;
+ /** Thread pool object used for multi-thread class translation. */
+ private static ExecutorService classTranslatorPool;
- /** used to handle Errors for multi-threaded file processing */
- private static List<Future<Void>> parallelProcessorFutures;
+ /** Single thread executor, for collecting results of parallel translation,
+ * and adding classes to dex file in original input file order. */
+ private static ExecutorService classDefItemConsumer;
+
+ /** Futures for {@code classDefItemConsumer} tasks. */
+ private static List<Future<Boolean>> addToDexFutures =
+ new ArrayList<Future<Boolean>>();
+
+ /** Thread pool object used for multi-thread dex conversion (to byte array).
+ * Used in combination with multi-dex support, to allow outputing
+ * a completed dex file, in parallel with continuing processing. */
+ private static ExecutorService dexOutPool;
+
+ /** Futures for {@code dexOutPool} task. */
+ private static List<Future<byte[]>> dexOutputFutures =
+ new ArrayList<Future<byte[]>>();
+
+ /** Lock object used to to coordinate dex file rotation, and
+ * multi-threaded translation. */
+ private static Object dexRotationLock = new Object();
+
+ /** Record the number if method indices "reserved" for files
+ * committed to translation in the context of the current dex
+ * file, but not yet added. */
+ private static int maxMethodIdsInProcess = 0;
+
+ /** Record the number if field indices "reserved" for files
+ * committed to translation in the context of the current dex
+ * file, but not yet added. */
+ private static int maxFieldIdsInProcess = 0;
/** true if any files are successfully processed */
private static volatile boolean anyFilesProcessed;
@@ -224,6 +254,7 @@
* @return 0 if success > 0 otherwise.
*/
public static int run(Arguments arguments) throws IOException {
+
// Reset the error count to start fresh.
errors.set(0);
// empty the list, so that tools that load dx and keep it around
@@ -289,7 +320,7 @@
byte[] outArray = null;
if (!outputDex.isEmpty() || (args.humanOutName != null)) {
- outArray = writeDex();
+ outArray = writeDex(outputDex);
if (outArray == null) {
return 2;
@@ -324,13 +355,14 @@
private static int runMultiDex() throws IOException {
assert !args.incremental;
- assert args.numThreads == 1;
if (args.mainDexListFile != null) {
classesInMainDex = new HashSet<String>();
readPathsFromFile(args.mainDexListFile, classesInMainDex);
}
+ dexOutPool = Executors.newFixedThreadPool(args.numThreads);
+
if (!processAllFiles()) {
return 1;
}
@@ -341,14 +373,31 @@
if (outputDex != null) {
// this array is null if no classes were defined
- dexOutputArrays.add(writeDex());
+
+ dexOutputFutures.add(dexOutPool.submit(new DexWriter(outputDex)));
// Effectively free up the (often massive) DexFile memory.
outputDex = null;
}
+ try {
+ dexOutPool.shutdown();
+ if (!dexOutPool.awaitTermination(600L, TimeUnit.SECONDS)) {
+ throw new RuntimeException("Timed out waiting for dex writer threads.");
+ }
+
+ for (Future<byte[]> f : dexOutputFutures) {
+ dexOutputArrays.add(f.get());
+ }
+
+ } catch (InterruptedException ex) {
+ dexOutPool.shutdownNow();
+ throw new RuntimeException("A dex writer thread has been interrupted.");
+ } catch (Exception e) {
+ dexOutPool.shutdownNow();
+ throw new RuntimeException("Unexpected exception in dex writer thread");
+ }
if (args.jarOutput) {
-
for (int i = 0; i < dexOutputArrays.size(); i++) {
outputResources.put(getDexFileName(i),
dexOutputArrays.get(i));
@@ -368,7 +417,6 @@
closeOutput(out);
}
}
-
}
return 0;
@@ -474,10 +522,14 @@
anyFilesProcessed = false;
String[] fileNames = args.fileNames;
- if (args.numThreads > 1) {
- threadPool = Executors.newFixedThreadPool(args.numThreads);
- parallelProcessorFutures = new ArrayList<Future<Void>>();
- }
+ // translate classes in parallel
+ classTranslatorPool = new ThreadPoolExecutor(args.numThreads,
+ args.numThreads, 0, TimeUnit.SECONDS,
+ new ArrayBlockingQueue<Runnable>(2 * args.numThreads, true),
+ new ThreadPoolExecutor.CallerRunsPolicy());
+ // collect translated and write to dex in order
+ classDefItemConsumer = Executors.newSingleThreadExecutor();
+
try {
if (args.mainDexListFile != null) {
@@ -490,14 +542,26 @@
processOne(fileNames[i], mainPassFilter);
}
- if (dexOutputArrays.size() > 0) {
+ if (dexOutputFutures.size() > 0) {
throw new DexException("Too many classes in " + Arguments.MAIN_DEX_LIST_OPTION
+ ", main dex capacity exceeded");
}
if (args.minimalMainDex) {
// start second pass directly in a secondary dex file.
- createDexFile();
+
+ // Wait for classes in progress to complete
+ synchronized(dexRotationLock) {
+ while(maxMethodIdsInProcess > 0 || maxFieldIdsInProcess > 0) {
+ try {
+ dexRotationLock.wait();
+ } catch(InterruptedException ex) {
+ /* ignore */
+ }
+ }
+ }
+
+ rotateDexFile();
}
// remaining files
@@ -517,35 +581,36 @@
*/
}
- if (args.numThreads > 1) {
- try {
- threadPool.shutdown();
- if (!threadPool.awaitTermination(600L, TimeUnit.SECONDS)) {
- throw new RuntimeException("Timed out waiting for threads.");
+ try {
+ classTranslatorPool.shutdown();
+ classTranslatorPool.awaitTermination(600L, TimeUnit.SECONDS);
+ classDefItemConsumer.shutdown();
+ classDefItemConsumer.awaitTermination(600L, TimeUnit.SECONDS);
+
+ for (Future<Boolean> f : addToDexFutures) {
+ try {
+ f.get();
+ } catch(ExecutionException ex) {
+ // Catch any previously uncaught exceptions from
+ // class translation and adding to dex.
+ int count = errors.incrementAndGet();
+ if (count < 10) {
+ DxConsole.err.println("Uncaught translation error: " + ex.getCause());
+ } else {
+ throw new InterruptedException("Too many errors");
+ }
}
- } catch (InterruptedException ex) {
- threadPool.shutdownNow();
- throw new RuntimeException("A thread has been interrupted.");
}
- try {
- for (Future<?> future : parallelProcessorFutures) {
- future.get();
- }
- } catch (ExecutionException e) {
- Throwable cause = e.getCause();
- // All Exceptions should have been handled in the ParallelProcessor, only Errors
- // should remain
- if (cause instanceof Error) {
- throw (Error) e.getCause();
- } else {
- throw new AssertionError(e.getCause());
- }
- } catch (InterruptedException e) {
- // If we're here, it means all threads have completed cleanly, so there should not be
- // any InterruptedException
- throw new AssertionError(e);
- }
+ } catch (InterruptedException ie) {
+ classTranslatorPool.shutdownNow();
+ classDefItemConsumer.shutdownNow();
+ throw new RuntimeException("Translation has been interrupted", ie);
+ } catch (Exception e) {
+ classTranslatorPool.shutdownNow();
+ classDefItemConsumer.shutdownNow();
+ e.printStackTrace(System.out);
+ throw new RuntimeException("Unexpected exception in translator thread.", e);
}
int errorNum = errors.get();
@@ -572,10 +637,6 @@
}
private static void createDexFile() {
- if (outputDex != null) {
- dexOutputArrays.add(writeDex());
- }
-
outputDex = new DexFile(args.dexOptions);
if (args.dumpWidth != 0) {
@@ -583,6 +644,18 @@
}
}
+ private static void rotateDexFile() {
+ if (outputDex != null) {
+ if (dexOutPool != null) {
+ dexOutputFutures.add(dexOutPool.submit(new DexWriter(outputDex)));
+ } else {
+ dexOutputArrays.add(writeDex(outputDex));
+ }
+ }
+
+ createDexFile();
+ }
+
/**
* Processes one pathname element.
*
@@ -594,47 +667,18 @@
private static void processOne(String pathname, FileNameFilter filter) {
ClassPathOpener opener;
- opener = new ClassPathOpener(pathname, false, filter,
- new ClassPathOpener.Consumer() {
+ opener = new ClassPathOpener(pathname, false, filter, new FileBytesConsumer());
- @Override
- public boolean processFileBytes(String name, long lastModified, byte[] bytes) {
- return Main.processFileBytes(name, lastModified, bytes);
- }
-
- @Override
- public void onException(Exception ex) {
- if (ex instanceof StopProcessing) {
- throw (StopProcessing) ex;
- } else if (ex instanceof SimException) {
- DxConsole.err.println("\nEXCEPTION FROM SIMULATION:");
- DxConsole.err.println(ex.getMessage() + "\n");
- DxConsole.err.println(((SimException) ex).getContext());
- } else {
- DxConsole.err.println("\nUNEXPECTED TOP-LEVEL EXCEPTION:");
- ex.printStackTrace(DxConsole.err);
- }
- errors.incrementAndGet();
- }
-
- @Override
- public void onProcessArchiveStart(File file) {
- if (args.verbose) {
- DxConsole.out.println("processing archive " + file +
- "...");
- }
- }
- });
-
- if (args.numThreads > 1) {
- parallelProcessorFutures.add(threadPool.submit(new ParallelProcessor(opener)));
- } else {
- if (opener.process()) {
- anyFilesProcessed = true;
- }
+ if (opener.process()) {
+ updateStatus(true);
}
}
+ private static void updateStatus(boolean res) {
+ anyFilesProcessed |= res;
+ }
+
+
/**
* Processes one file, which may be either a class or a resource.
*
@@ -643,6 +687,7 @@
* @return whether processing was successful
*/
private static boolean processFileBytes(String name, long lastModified, byte[] bytes) {
+
boolean isClass = name.endsWith(".class");
boolean isClassesDex = name.equals(DexFormat.DEX_IN_JAR_NAME);
boolean keepResources = (outputResources != null);
@@ -670,7 +715,10 @@
if (lastModified < minimumFileAge) {
return true;
}
- return processClass(fixedName, bytes);
+ processClass(fixedName, bytes);
+ // Assume that an exception may occur. Status will be updated
+ // asynchronously, if the class compiles without error.
+ return false;
} else if (isClassesDex) {
synchronized (libraryDexBuffers) {
libraryDexBuffers.add(bytes);
@@ -697,42 +745,30 @@
checkClassName(name);
}
- DirectClassFile cf =
- new DirectClassFile(bytes, name, args.cfOptions.strictNameCheck);
-
- cf.setAttributeFactory(StdAttributeFactory.THE_ONE);
- cf.getMagic();
-
- int numMethodIds = outputDex.getMethodIds().items().size();
- int numFieldIds = outputDex.getFieldIds().items().size();
- int constantPoolSize = cf.getConstantPool().size();
-
- int maxMethodIdsInDex = numMethodIds + constantPoolSize + cf.getMethods().size() +
- MAX_METHOD_ADDED_DURING_DEX_CREATION;
- int maxFieldIdsInDex = numFieldIds + constantPoolSize + cf.getFields().size() +
- MAX_FIELD_ADDED_DURING_DEX_CREATION;
-
- if (args.multiDex
- // Never switch to the next dex if current dex is already empty
- && (outputDex.getClassDefs().items().size() > 0)
- && ((maxMethodIdsInDex > args.maxNumberOfIdxPerDex) ||
- (maxFieldIdsInDex > args.maxNumberOfIdxPerDex))) {
- DexFile completeDex = outputDex;
- createDexFile();
- assert (completeDex.getMethodIds().items().size() <= numMethodIds +
- MAX_METHOD_ADDED_DURING_DEX_CREATION) &&
- (completeDex.getFieldIds().items().size() <= numFieldIds +
- MAX_FIELD_ADDED_DURING_DEX_CREATION);
+ try {
+ new DirectClassFileConsumer(name, bytes, null).call(
+ new ClassParserTask(name, bytes).call());
+ } catch(Exception ex) {
+ throw new RuntimeException("Exception parsing classes", ex);
}
- try {
- ClassDefItem clazz =
- CfTranslator.translate(cf, bytes, args.cfOptions, args.dexOptions, outputDex);
- synchronized (outputDex) {
- outputDex.add(clazz);
- }
- return true;
+ return true;
+ }
+
+ private static DirectClassFile parseClass(String name, byte[] bytes) {
+
+ DirectClassFile cf = new DirectClassFile(bytes, name,
+ args.cfOptions.strictNameCheck);
+ cf.setAttributeFactory(StdAttributeFactory.THE_ONE);
+ cf.getMagic(); // triggers the actual parsing
+ return cf;
+ }
+
+ private static ClassDefItem translateClass(byte[] bytes, DirectClassFile cf) {
+ try {
+ return CfTranslator.translate(cf, bytes, args.cfOptions,
+ args.dexOptions, outputDex);
} catch (ParseException ex) {
DxConsole.err.println("\ntrouble processing:");
if (args.debug) {
@@ -742,7 +778,14 @@
}
}
errors.incrementAndGet();
- return false;
+ return null;
+ }
+
+ private static boolean addClassToDex(ClassDefItem clazz) {
+ synchronized (outputDex) {
+ outputDex.add(clazz);
+ }
+ return true;
}
/**
@@ -792,7 +835,7 @@
* @return {@code null-ok;} the converted {@code byte[]} or {@code null}
* if there was a problem
*/
- private static byte[] writeDex() {
+ private static byte[] writeDex(DexFile outputDex) {
byte[] outArray = null;
try {
@@ -831,7 +874,6 @@
}
return null;
}
-
return outArray;
}
@@ -1199,6 +1241,9 @@
/** whether to run in debug mode */
public boolean debug = false;
+ /** whether to emit warning messages */
+ public boolean warnings = true;
+
/** whether to emit high-level verbose human-oriented output */
public boolean verbose = false;
@@ -1409,6 +1454,8 @@
while(parser.getNext()) {
if (parser.isArg("--debug")) {
debug = true;
+ } else if (parser.isArg("--no-warning")) {
+ warnings = false;
} else if (parser.isArg("--verbose")) {
verbose = true;
} else if (parser.isArg("--verbose-dump")) {
@@ -1542,12 +1589,6 @@
throw new UsageException();
}
- if (multiDex && numThreads != 1) {
- System.out.println(NUM_THREADS_OPTION + " is ignored when used with "
- + MULTI_DEX_OPTION);
- numThreads = 1;
- }
-
if (multiDex && incremental) {
System.err.println(INCREMENTAL_OPTION + " is not supported with "
+ MULTI_DEX_OPTION);
@@ -1580,28 +1621,272 @@
cfOptions.optimizeListFile = optimizeListFile;
cfOptions.dontOptimizeListFile = dontOptimizeListFile;
cfOptions.statistics = statistics;
- cfOptions.warn = DxConsole.err;
+
+ if (warnings) {
+ cfOptions.warn = DxConsole.err;
+ } else {
+ cfOptions.warn = DxConsole.noop;
+ }
dexOptions = new DexOptions();
dexOptions.forceJumbo = forceJumbo;
}
}
- /** Callable helper class to process files in multiple threads */
- private static class ParallelProcessor implements Callable<Void> {
+ /**
+ * Callback class for processing input file bytes, produced by the
+ * ClassPathOpener.
+ */
+ private static class FileBytesConsumer implements ClassPathOpener.Consumer {
- ClassPathOpener classPathOpener;
-
- private ParallelProcessor(ClassPathOpener classPathOpener) {
- this.classPathOpener = classPathOpener;
+ @Override
+ public boolean processFileBytes(String name, long lastModified,
+ byte[] bytes) {
+ return Main.processFileBytes(name, lastModified, bytes);
}
@Override
- public Void call() throws Exception {
- if (classPathOpener.process()) {
- anyFilesProcessed = true;
+ public void onException(Exception ex) {
+ if (ex instanceof StopProcessing) {
+ throw (StopProcessing) ex;
+ } else if (ex instanceof SimException) {
+ DxConsole.err.println("\nEXCEPTION FROM SIMULATION:");
+ DxConsole.err.println(ex.getMessage() + "\n");
+ DxConsole.err.println(((SimException) ex).getContext());
+ } else {
+ DxConsole.err.println("\nUNEXPECTED TOP-LEVEL EXCEPTION:");
+ ex.printStackTrace(DxConsole.err);
}
- return null;
+ errors.incrementAndGet();
+ }
+
+ @Override
+ public void onProcessArchiveStart(File file) {
+ if (args.verbose) {
+ DxConsole.out.println("processing archive " + file + "...");
+ }
+ }
+ }
+
+ /** Callable helper class to parse class bytes. */
+ private static class ClassParserTask implements Callable<DirectClassFile> {
+
+ String name;
+ byte[] bytes;
+
+ private ClassParserTask(String name, byte[] bytes) {
+ this.name = name;
+ this.bytes = bytes;
+ }
+
+ @Override
+ public DirectClassFile call() throws Exception {
+ DirectClassFile cf = parseClass(name, bytes);
+
+ return cf;
+ }
+ }
+
+ /**
+ * Callable helper class used to sequentially collect the results of
+ * the (optionally parallel) translation phase, in correct input file order.
+ * This class is also responsible for coordinating dex file rotation
+ * with the ClassDefItemConsumer class.
+ * We maintain invariant that the number of indices used in the current
+ * dex file plus the max number of indices required by classes passed to
+ * the translation phase and not yet added to the dex file, is less than
+ * or equal to the dex file limit.
+ * For each parsed file, we estimate the maximum number of indices it may
+ * require. If passing the file to the translation phase would invalidate
+ * the invariant, we wait, until the next class is added to the dex file,
+ * and then reevaluate the invariant. If there are no further classes in
+ * the translation phase, we rotate the dex file.
+ */
+ private static class DirectClassFileConsumer implements Callable<Boolean> {
+
+ String name;
+ byte[] bytes;
+ Future<DirectClassFile> dcff;
+
+ private DirectClassFileConsumer(String name, byte[] bytes,
+ Future<DirectClassFile> dcff) {
+ this.name = name;
+ this.bytes = bytes;
+ this.dcff = dcff;
+ }
+
+ @Override
+ public Boolean call() throws Exception {
+
+ DirectClassFile cf = dcff.get();
+ return call(cf);
+ }
+
+ private Boolean call(DirectClassFile cf) {
+
+ int maxMethodIdsInClass = 0;
+ int maxFieldIdsInClass = 0;
+
+ if (args.multiDex) {
+
+ // Calculate max number of indices this class will add to the
+ // dex file.
+ // The possibility of overloading means that we can't easily
+ // know how many constant are needed for declared methods and
+ // fields. We therefore make the simplifying assumption that
+ // all constants are external method or field references.
+
+ int constantPoolSize = cf.getConstantPool().size();
+ maxMethodIdsInClass = constantPoolSize + cf.getMethods().size()
+ + MAX_METHOD_ADDED_DURING_DEX_CREATION;
+ maxFieldIdsInClass = constantPoolSize + cf.getFields().size()
+ + MAX_FIELD_ADDED_DURING_DEX_CREATION;
+ synchronized(dexRotationLock) {
+
+ int numMethodIds;
+ int numFieldIds;
+ // Number of indices used in current dex file.
+ synchronized(outputDex) {
+ numMethodIds = outputDex.getMethodIds().items().size();
+ numFieldIds = outputDex.getFieldIds().items().size();
+ }
+ // Wait until we're sure this class will fit in the current
+ // dex file.
+ while(((numMethodIds + maxMethodIdsInClass + maxMethodIdsInProcess
+ > args.maxNumberOfIdxPerDex) ||
+ (numFieldIds + maxFieldIdsInClass + maxFieldIdsInProcess
+ > args.maxNumberOfIdxPerDex))) {
+
+ if (maxMethodIdsInProcess > 0 || maxFieldIdsInProcess > 0) {
+ // There are classes in the translation phase that
+ // have not yet been added to the dex file, so we
+ // wait for the next class to complete.
+ try {
+ dexRotationLock.wait();
+ } catch(InterruptedException ex) {
+ /* ignore */
+ }
+ } else if (outputDex.getClassDefs().items().size() > 0) {
+ // There are no further classes in the translation
+ // phase, and we have a full dex file. Rotate!
+ rotateDexFile();
+ } else {
+ // The estimated number of indices is too large for
+ // an empty dex file. We proceed hoping the actual
+ // number of indices needed will fit.
+ break;
+ }
+ synchronized(outputDex) {
+ numMethodIds = outputDex.getMethodIds().items().size();
+ numFieldIds = outputDex.getFieldIds().items().size();
+ }
+ }
+ // Add our estimate to the total estimate for
+ // classes under translation.
+ maxMethodIdsInProcess += maxMethodIdsInClass;
+ maxFieldIdsInProcess += maxFieldIdsInClass;
+ }
+ }
+
+ // Submit class to translation phase.
+ Future<ClassDefItem> cdif = classTranslatorPool.submit(
+ new ClassTranslatorTask(name, bytes, cf));
+ Future<Boolean> res = classDefItemConsumer.submit(new ClassDefItemConsumer(
+ name, cdif, maxMethodIdsInClass, maxFieldIdsInClass));
+ addToDexFutures.add(res);
+
+ return true;
+ }
+ }
+
+
+ /** Callable helper class to translate classes in parallel */
+ private static class ClassTranslatorTask implements Callable<ClassDefItem> {
+
+ String name;
+ byte[] bytes;
+ DirectClassFile classFile;
+
+ private ClassTranslatorTask(String name, byte[] bytes,
+ DirectClassFile classFile) {
+ this.name = name;
+ this.bytes = bytes;
+ this.classFile = classFile;
+ }
+
+ @Override
+ public ClassDefItem call() {
+ ClassDefItem clazz = translateClass(bytes, classFile);
+ return clazz;
+ }
+ }
+
+ /**
+ * Callable helper class used to collect the results of
+ * the parallel translation phase, adding the translated classes to
+ * the current dex file in correct (deterministic) file order.
+ * This class is also responsible for coordinating dex file rotation
+ * with the DirectClassFileConsumer class.
+ */
+ private static class ClassDefItemConsumer implements Callable<Boolean> {
+
+ String name;
+ Future<ClassDefItem> futureClazz;
+ int maxMethodIdsInClass;
+ int maxFieldIdsInClass;
+
+ private ClassDefItemConsumer(String name, Future<ClassDefItem> futureClazz,
+ int maxMethodIdsInClass, int maxFieldIdsInClass) {
+ this.name = name;
+ this.futureClazz = futureClazz;
+ this.maxMethodIdsInClass = maxMethodIdsInClass;
+ this.maxFieldIdsInClass = maxFieldIdsInClass;
+ }
+
+ @Override
+ public Boolean call() throws Exception {
+ try {
+ ClassDefItem clazz = futureClazz.get();
+ if (clazz != null) {
+ addClassToDex(clazz);
+ updateStatus(true);
+ }
+ return true;
+ } catch(ExecutionException ex) {
+ // Rethrow previously uncaught translation exceptions.
+ // These, as well as any exceptions from addClassToDex,
+ // are handled and reported in processAllFiles().
+ Throwable t = ex.getCause();
+ throw (t instanceof Exception) ? (Exception) t : ex;
+ } finally {
+ if (args.multiDex) {
+ // Having added our actual indicies to the dex file,
+ // we subtract our original estimate from the total estimate,
+ // and signal the translation phase, which may be paused
+ // waiting to determine if more classes can be added to the
+ // current dex file, or if a new dex file must be created.
+ synchronized(dexRotationLock) {
+ maxMethodIdsInProcess -= maxMethodIdsInClass;
+ maxFieldIdsInProcess -= maxFieldIdsInClass;
+ dexRotationLock.notifyAll();
+ }
+ }
+ }
+ }
+ }
+
+ /** Callable helper class to convert dex files in worker threads */
+ private static class DexWriter implements Callable<byte[]> {
+
+ private DexFile dexFile;
+
+ private DexWriter(DexFile dexFile) {
+ this.dexFile = dexFile;
+ }
+
+ @Override
+ public byte[] call() throws IOException {
+ return writeDex(dexFile);
}
}
}
diff --git a/dx/src/com/android/dx/dex/file/MixedItemSection.java b/dx/src/com/android/dx/dex/file/MixedItemSection.java
index 4edc6b6..9053043 100644
--- a/dx/src/com/android/dx/dex/file/MixedItemSection.java
+++ b/dx/src/com/android/dx/dex/file/MixedItemSection.java
@@ -189,7 +189,7 @@
* @param item {@code non-null;} the item to intern
* @return {@code non-null;} the equivalent interned instance
*/
- public <T extends OffsettedItem> T intern(T item) {
+ public synchronized <T extends OffsettedItem> T intern(T item) {
throwIfPrepared();
OffsettedItem result = interns.get(item);
diff --git a/dx/src/com/android/dx/dex/file/ProtoIdsSection.java b/dx/src/com/android/dx/dex/file/ProtoIdsSection.java
index 4b1303b..b7df10c 100644
--- a/dx/src/com/android/dx/dex/file/ProtoIdsSection.java
+++ b/dx/src/com/android/dx/dex/file/ProtoIdsSection.java
@@ -86,7 +86,7 @@
* @param prototype {@code non-null;} the prototype to intern
* @return {@code non-null;} the interned reference
*/
- public ProtoIdItem intern(Prototype prototype) {
+ public synchronized ProtoIdItem intern(Prototype prototype) {
if (prototype == null) {
throw new NullPointerException("prototype == null");
}
diff --git a/dx/src/com/android/dx/dex/file/StringIdsSection.java b/dx/src/com/android/dx/dex/file/StringIdsSection.java
index 7aff82e..6826c5a 100644
--- a/dx/src/com/android/dx/dex/file/StringIdsSection.java
+++ b/dx/src/com/android/dx/dex/file/StringIdsSection.java
@@ -117,7 +117,7 @@
* @param string {@code non-null;} the string to intern
* @return {@code non-null;} the interned string
*/
- public StringIdItem intern(StringIdItem string) {
+ public synchronized StringIdItem intern(StringIdItem string) {
if (string == null) {
throw new NullPointerException("string == null");
}
@@ -140,7 +140,7 @@
*
* @param nat {@code non-null;} the name-and-type
*/
- public void intern(CstNat nat) {
+ public synchronized void intern(CstNat nat) {
intern(nat.getName());
intern(nat.getDescriptor());
}
diff --git a/dx/src/com/android/dx/dex/file/TypeIdsSection.java b/dx/src/com/android/dx/dex/file/TypeIdsSection.java
index c3380f4..35d8e66 100644
--- a/dx/src/com/android/dx/dex/file/TypeIdsSection.java
+++ b/dx/src/com/android/dx/dex/file/TypeIdsSection.java
@@ -106,7 +106,7 @@
* @param type {@code non-null;} the type to intern
* @return {@code non-null;} the interned reference
*/
- public TypeIdItem intern(Type type) {
+ public synchronized TypeIdItem intern(Type type) {
if (type == null) {
throw new NullPointerException("type == null");
}
diff --git a/dx/tests/129-numthread-deterministic/expected.txt b/dx/tests/129-numthread-deterministic/expected.txt
new file mode 100644
index 0000000..5418338
--- /dev/null
+++ b/dx/tests/129-numthread-deterministic/expected.txt
@@ -0,0 +1 @@
+Yay!
diff --git a/dx/tests/129-numthread-deterministic/info.txt b/dx/tests/129-numthread-deterministic/info.txt
new file mode 100644
index 0000000..e4885fa
--- /dev/null
+++ b/dx/tests/129-numthread-deterministic/info.txt
@@ -0,0 +1,2 @@
+Test that dx generates deterministic output when using --num-threads
+
diff --git a/dx/tests/129-numthread-deterministic/run b/dx/tests/129-numthread-deterministic/run
new file mode 100644
index 0000000..fdc0506
--- /dev/null
+++ b/dx/tests/129-numthread-deterministic/run
@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Stop if something fails.
+set -e
+
+# Write out classes
+mkdir src
+awk '
+BEGIN {
+ for (c = 1; c <= 500; c++) {
+ writeClass(c);
+ }
+}
+function writeClass(name) {
+ fileName = "src/Clazz" name ".java";
+ printf("public class Clazz%s {\n", name) > fileName;
+ for (i = 1; i <= 100; i++) {
+ printf(" int field%d;\n", i) > fileName;
+ }
+ for (i = 1; i <= 100; i++) {
+ printf(" void method%d(int param) { }\n", i) > fileName;
+ }
+ printf("}\n") > fileName;
+}'
+
+
+mkdir classes
+${JAVAC} -d classes `find src -name '*.java'`
+mkdir out
+dx -JXmx4g -JXms4g --dex --no-optimize --output=out classes
+mkdir out-multi
+dx -JXmx4g -JXms4g --dex --no-optimize --num-threads=4 --output=out-multi classes
+diff -r out out-multi > unit-out.txt
+
+if [ "$?" = "0" ]; then
+ echo "Yay!"
+else
+ cat unit-out.txt
+fi
+
diff --git a/dx/tests/130-numthread-multidex-deterministic/expected.txt b/dx/tests/130-numthread-multidex-deterministic/expected.txt
new file mode 100644
index 0000000..5418338
--- /dev/null
+++ b/dx/tests/130-numthread-multidex-deterministic/expected.txt
@@ -0,0 +1 @@
+Yay!
diff --git a/dx/tests/130-numthread-multidex-deterministic/info.txt b/dx/tests/130-numthread-multidex-deterministic/info.txt
new file mode 100644
index 0000000..e4885fa
--- /dev/null
+++ b/dx/tests/130-numthread-multidex-deterministic/info.txt
@@ -0,0 +1,2 @@
+Test that dx generates deterministic output when using --num-threads
+
diff --git a/dx/tests/130-numthread-multidex-deterministic/run b/dx/tests/130-numthread-multidex-deterministic/run
new file mode 100644
index 0000000..b43ba1f
--- /dev/null
+++ b/dx/tests/130-numthread-multidex-deterministic/run
@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Stop if something fails.
+set -e
+
+# Write out classes
+mkdir src
+awk '
+BEGIN {
+ for (c = 1; c <= 1000; c++) {
+ writeClass(c);
+ }
+}
+function writeClass(name) {
+ fileName = "src/Clazz" name ".java";
+ printf("public class Clazz%s {\n", name) > fileName;
+ for (i = 1; i <= 100; i++) {
+ printf(" int field%d;\n", i) > fileName;
+ }
+ for (i = 1; i <= 100; i++) {
+ printf(" void method%d(int param) { }\n", i) > fileName;
+ }
+ printf("}\n") > fileName;
+}'
+
+
+mkdir classes
+${JAVAC} -d classes `find src -name '*.java'`
+mkdir out
+dx -JXmx4g -JXms4g --dex --no-optimize --multi-dex --output=out classes
+mkdir out-multi
+dx -JXmx4g -JXms4g --dex --no-optimize --multi-dex --num-threads=4 --output=out-multi classes
+diff -r out out-multi > unit-out.txt
+
+if [ "$?" = "0" ]; then
+ echo "Yay!"
+else
+ cat unit-out.txt
+fi
+
diff --git a/dx/tests/131-perf/ClassGen.java b/dx/tests/131-perf/ClassGen.java
new file mode 100644
index 0000000..c35bbcf
--- /dev/null
+++ b/dx/tests/131-perf/ClassGen.java
@@ -0,0 +1,53 @@
+import java.io.File;
+import java.io.PrintWriter;
+
+public class ClassGen {
+
+ public static void main(String... args) {
+
+ int start = 1;
+ int end = 8024;
+ int fields = 4;
+ int methods = 6;
+ if (args.length > 0) {
+ start = Integer.parseInt(args[0]);
+ }
+ if (args.length > 1) {
+ end = Integer.parseInt(args[1]);
+ }
+ if (args.length > 2) {
+ fields = Integer.parseInt(args[2]);
+ }
+ if (args.length > 3) {
+ methods = Integer.parseInt(args[3]);
+ }
+
+ for (int file = start; file <= end; file++) {
+ try {
+ File f = new File("src/Clazz" + file + ".java");
+ PrintWriter pw = new PrintWriter(f);
+ pw.println("class Clazz" + file + " {");
+ for (int field = 1; field <= fields; field++) {
+ pw.println(" public static int f" + field + ";");
+ }
+ for (int method = 1; method <= methods; method++) {
+ pw.println(" boolean m" + method + "_" + (file%(end/2)) + "() {"
+);
+ pw.println(" int max = Thread.MAX_PRIORITY;");
+ pw.println(" for (int i = 0; i < max; i++) {");
+ pw.println(" System.out.println(\"Hello from: \" + Clazz"
+ + file + ".class + \".method" + method
+ + "() \" + Clazz" + (end-file+1) + ".f1);");
+ pw.println(" Thread.dumpStack();");
+ pw.println(" }");
+ pw.println(" return Thread.holdsLock(this);");
+ pw.println(" }");
+ }
+ pw.println("}");
+ pw.close();
+ } catch(Exception ex) {
+ System.out.println("Ups");
+ }
+ }
+ }
+}
diff --git a/dx/tests/131-perf/expected.txt b/dx/tests/131-perf/expected.txt
new file mode 100644
index 0000000..5418338
--- /dev/null
+++ b/dx/tests/131-perf/expected.txt
@@ -0,0 +1 @@
+Yay!
diff --git a/dx/tests/131-perf/info.txt b/dx/tests/131-perf/info.txt
new file mode 100644
index 0000000..f5b6a0c
--- /dev/null
+++ b/dx/tests/131-perf/info.txt
@@ -0,0 +1,2 @@
+Script for --multi-dex --num-threads performance testing, default just test options can be used together
+
diff --git a/dx/tests/131-perf/run b/dx/tests/131-perf/run
new file mode 100644
index 0000000..e57545c
--- /dev/null
+++ b/dx/tests/131-perf/run
@@ -0,0 +1,88 @@
+#!/bin/bash
+#
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Stop if something fails.
+set -e
+
+# Write out classes
+
+${JAVAC} ClassGen.java
+
+mkdir src
+mkdir classes
+
+# Heap size, min and max
+MEM=4g
+
+MULTIDEX="--multi-dex"
+THREADS="--num-threads=5"
+
+# Extra statistics, use to calibrate test.
+#EXTRA="--profile-concurrency"
+
+# Test smaller dex files
+#EXTRA="--set-max-idx-number=20000"
+
+# Test GC options
+#GC="-JXX:+UseConcMarkSweepGC"
+
+# Limit HW threads
+#TASKSET="taskset 0x00000fff
+
+# Number of classes, initial
+TEST_SIZE=1000
+
+# number of classes, max
+LIMIT=1000
+
+# Number of additional classes per test
+STEP=100
+
+# Number of fields per classes
+FIELDS=4
+
+# Number of methods per class
+METHODS=6
+
+
+first=1;
+while [ $TEST_SIZE -le $LIMIT ]; do
+ rm -rf out
+ mkdir out
+
+ sleep 2
+ java -classpath . ClassGen $first $TEST_SIZE $FIELDS $METHODS
+ first=`expr $TEST_SIZE + 1`
+
+ ${JAVAC} -d classes `find src -name '*.java'`
+ (cd classes; jar cf ../x.jar `find . -name '*.class'`)
+ sleep 3
+
+ start=`date +'%s%N'`
+ $TASKSET dx -JXmx$MEM -JXms$MEM $GC --dex $EXTRA --no-optimize $MULTIDEX $THREADS --output=out x.jar
+ end=`date +'%s%N'`
+ nsec=`expr $end - $start`
+ msec=`expr $nsec / 1000000`
+ echo "Classes/msec $TEST_SIZE $msec"
+
+ TEST_SIZE=`expr $TEST_SIZE + $STEP`
+done
+
+if [ "$?" = "1" ]; then
+ echo "Yay!"
+else
+ cat unit-out.txt
+fi
