Feature: Parallel Building (#3476)

* perf: process smali code in parallel

Note: backsmali can't be properly multithreaded because of the synchronized methods inside

* perf: start backsmali concurrently with a resources decompiler

* perf: speed up apk building by skipping temp archive creation

Now we're not compressing the same data twice

* refactor: extract duplicated code

* refactor: rename methods and inline some comments

brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java

@@ -42,17 +42,19 @@ import javax.xml.transform.TransformerException;
 import java.io.*;
 import java.nio.file.Files;
 import java.util.*;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.logging.Logger;
 import java.util.zip.CRC32;
 import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
 import java.util.zip.ZipOutputStream;
 
 public class ApkBuilder {
     private final static Logger LOGGER = Logger.getLogger(ApkBuilder.class.getName());
 
+    private final AtomicReference<AndrolibException> mBuildError = new AtomicReference<>(null);
     private final Config mConfig;
     private final ExtFile mApkDir;
+    private BackgroundWorker mWorker;
     private ApkInfo mApkInfo;
     private int mMinSdkVersion = 0;
 
@@ -78,51 +80,55 @@ public class ApkBuilder {
 
     public void build(File outFile) throws BrutException {
         LOGGER.info("Using Apktool " + ApktoolProperties.getVersion());
+        try {
+            mWorker = new BackgroundWorker();
+            mApkInfo = ApkInfo.load(mApkDir);
 
-        mApkInfo = ApkInfo.load(mApkDir);
+            if (mApkInfo.getSdkInfo() != null && mApkInfo.getSdkInfo().get("minSdkVersion") != null) {
-
+                String minSdkVersion = mApkInfo.getSdkInfo().get("minSdkVersion");
-        if (mApkInfo.getSdkInfo() != null && mApkInfo.getSdkInfo().get("minSdkVersion") != null) {
+                mMinSdkVersion = mApkInfo.getMinSdkVersionFromAndroidCodename(minSdkVersion);
-            String minSdkVersion = mApkInfo.getSdkInfo().get("minSdkVersion");
-            mMinSdkVersion = mApkInfo.getMinSdkVersionFromAndroidCodename(minSdkVersion);
-        }
-
-        if (outFile == null) {
-            String outFileName = mApkInfo.apkFileName;
-            outFile = new File(mApkDir, "dist" + File.separator + (outFileName == null ? "out.apk" : outFileName));
-        }
-
-        //noinspection ResultOfMethodCallIgnored
-        new File(mApkDir, APK_DIRNAME).mkdirs();
-        File manifest = new File(mApkDir, "AndroidManifest.xml");
-        File manifestOriginal = new File(mApkDir, "AndroidManifest.xml.orig");
-
-        buildSources();
-        buildNonDefaultSources();
-        buildManifestFile(manifest, manifestOriginal);
-        buildResources();
-        buildLibs();
-        buildCopyOriginalFiles();
-        buildApk(outFile);
-
-        // we must go after the Apk is built, and copy the files in via Zip
-        // this is because Aapt won't add files it doesn't know (ex unknown files)
-        buildUnknownFiles(outFile);
-
-        // we copied the AndroidManifest.xml to AndroidManifest.xml.orig so we can edit it
-        // lets restore the unedited one, to not change the original
-        if (manifest.isFile() && manifest.exists() && manifestOriginal.isFile()) {
-            try {
-                if (new File(mApkDir, "AndroidManifest.xml").delete()) {
-                    FileUtils.moveFile(manifestOriginal, manifest);
-                }
-            } catch (IOException ex) {
-                throw new AndrolibException(ex.getMessage());
             }
+
+            if (outFile == null) {
+                String outFileName = mApkInfo.apkFileName;
+                outFile = new File(mApkDir, "dist" + File.separator + (outFileName == null ? "out.apk" : outFileName));
+            }
+
+            //noinspection ResultOfMethodCallIgnored
+            new File(mApkDir, APK_DIRNAME).mkdirs();
+            File manifest = new File(mApkDir, "AndroidManifest.xml");
+            File manifestOriginal = new File(mApkDir, "AndroidManifest.xml.orig");
+
+            scheduleBuildDexFiles();
+            backupManifestFile(manifest, manifestOriginal);
+            buildResources();
+            copyLibs();
+            copyOriginalFilesIfEnabled();
+            mWorker.waitForFinish();
+            if (mBuildError.get() != null) {
+                throw mBuildError.get();
+            }
+
+            buildApk(outFile);
+
+            // we copied the AndroidManifest.xml to AndroidManifest.xml.orig so we can edit it
+            // lets restore the unedited one, to not change the original
+            if (manifest.isFile() && manifest.exists() && manifestOriginal.isFile()) {
+                try {
+                    if (new File(mApkDir, "AndroidManifest.xml").delete()) {
+                        FileUtils.moveFile(manifestOriginal, manifest);
+                    }
+                } catch (IOException ex) {
+                    throw new AndrolibException(ex.getMessage());
+                }
+            }
+            LOGGER.info("Built apk into: " + outFile.getPath());
+        } finally {
+            mWorker.shutdownNow();
         }
-        LOGGER.info("Built apk into: " + outFile.getPath());
     }
 
-    private void buildManifestFile(File manifest, File manifestOriginal) throws AndrolibException {
+    private void backupManifestFile(File manifest, File manifestOriginal) throws AndrolibException {
         // If we decoded in "raw", we cannot patch AndroidManifest
         if (new File(mApkDir, "resources.arsc").exists()) {
             return;
@@ -141,24 +147,17 @@ public class ApkBuilder {
         }
     }
 
-    private void buildSources() throws AndrolibException {
+    private void scheduleBuildDexFiles() throws AndrolibException {
-        if (!buildSourcesRaw("classes.dex") && !buildSourcesSmali("smali", "classes.dex")) {
-            LOGGER.warning("Could not find sources");
-        }
-    }
-
-    private void buildNonDefaultSources() throws AndrolibException {
         try {
+            mWorker.submit(() -> scheduleDexBuild("classes.dex", "smali"));
+
             // loop through any smali_ directories for multi-dex apks
             Map<String, Directory> dirs = mApkDir.getDirectory().getDirs();
             for (Map.Entry<String, Directory> directory : dirs.entrySet()) {
                 String name = directory.getKey();
                 if (name.startsWith("smali_")) {
                     String filename = name.substring(name.indexOf("_") + 1) + ".dex";
-
+                    mWorker.submit(() -> scheduleDexBuild(filename, name));
-                    if (!buildSourcesRaw(filename) && !buildSourcesSmali(name, filename)) {
-                        LOGGER.warning("Could not find sources");
-                    }
                 }
             }
 
@@ -177,6 +176,19 @@ public class ApkBuilder {
         }
     }
 
+    private void scheduleDexBuild(String filename, String smali) {
+        try {
+            if (mBuildError.get() != null) {
+                return;
+            }
+            if (!buildSourcesRaw(filename) && !buildSourcesSmali(smali, filename)) {
+                LOGGER.warning("Could not find sources");
+            }
+        } catch (AndrolibException e) {
+            mBuildError.compareAndSet(null, e);
+        }
+    }
+
     private boolean buildSourcesRaw(String filename) throws AndrolibException {
         File working = new File(mApkDir, filename);
         if (!working.exists()) {
@@ -214,6 +226,7 @@ public class ApkBuilder {
     }
 
     private void buildResources() throws BrutException {
+        // create res folder, manifest file and resources.arsc
         if (!buildResourcesRaw() && !buildResourcesFull() && !buildManifest()) {
             LOGGER.warning("Could not find resources");
         }
@@ -375,7 +388,7 @@ public class ApkBuilder {
         }
     }
 
-    private void buildLibs() throws AndrolibException {
+    private void copyLibs() throws AndrolibException {
         buildLibrary("lib");
         buildLibrary("libs");
         buildLibrary("kotlin");
@@ -401,7 +414,7 @@ public class ApkBuilder {
         }
     }
 
-    private void buildCopyOriginalFiles() throws AndrolibException {
+    private void copyOriginalFilesIfEnabled() throws AndrolibException {
         if (mConfig.copyOriginalFiles) {
             File originalDir = new File(mApkDir, "original");
             if (originalDir.exists()) {
@@ -427,49 +440,34 @@ public class ApkBuilder {
         }
     }
 
-    private void buildUnknownFiles(File outFile) throws AndrolibException {
+    private void buildApk(File outApk) throws AndrolibException {
-        if (mApkInfo.unknownFiles != null) {
+        LOGGER.info("Building apk file...");
-            LOGGER.info("Copying unknown files/dir...");
+        if (outApk.exists()) {
-
-            Map<String, String> files = mApkInfo.unknownFiles;
-            File tempFile = new File(outFile.getParent(), outFile.getName() + ".apktool_temp");
-            boolean renamed = outFile.renameTo(tempFile);
-            if (!renamed) {
-                throw new AndrolibException("Unable to rename temporary file");
-            }
-
-            try (
-                ZipFile inputFile = new ZipFile(tempFile);
-                ZipOutputStream actualOutput = new ZipOutputStream(Files.newOutputStream(outFile.toPath()))
-            ) {
-                copyExistingFiles(inputFile, actualOutput);
-                copyUnknownFiles(actualOutput, files);
-            } catch (IOException | BrutException ex) {
-                throw new AndrolibException(ex);
-            }
-
-            // Remove our temporary file.
             //noinspection ResultOfMethodCallIgnored
-            tempFile.delete();
+            outApk.delete();
-        }
+        } else {
-    }
+            File outDir = outApk.getParentFile();
-
+            if (outDir != null && !outDir.exists()) {
-    private void copyExistingFiles(ZipFile inputFile, ZipOutputStream outputFile) throws IOException {
+                //noinspection ResultOfMethodCallIgnored
-        // First, copy the contents from the existing outFile:
+                outDir.mkdirs();
-        Enumeration<? extends ZipEntry> entries = inputFile.entries();
-        while (entries.hasMoreElements()) {
-            ZipEntry entry = new ZipEntry(entries.nextElement());
-
-            // We can't reuse the compressed size because it depends on compression sizes.
-            entry.setCompressedSize(-1);
-            outputFile.putNextEntry(entry);
-
-            // No need to create directory entries in the final apk
-            if (!entry.isDirectory()) {
-                BrutIO.copy(inputFile, outputFile, entry);
             }
+        }
+        File assetDir = new File(mApkDir, "assets");
+        if (!assetDir.exists()) {
+            assetDir = null;
+        }
+        try (ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(outApk.toPath()))) {
+            // zip all AAPT-generated files
+            ZipUtils.zipFoldersPreserveStream(new File(mApkDir, APK_DIRNAME), zipOutputStream, assetDir, mApkInfo.doNotCompress);
 
-            outputFile.closeEntry();
+            // we must copy some files manually
+            // this is because Aapt won't add files it doesn't know (ex unknown files)
+            if (mApkInfo.unknownFiles != null) {
+                LOGGER.info("Copying unknown files/dir...");
+                copyUnknownFiles(zipOutputStream, mApkInfo.unknownFiles);
+            }
+        } catch (IOException | BrutException e) {
+            throw new AndrolibException(e);
         }
     }
 
@@ -513,33 +511,6 @@ public class ApkBuilder {
         }
     }
 
-    private void buildApk(File outApk) throws AndrolibException {
-        LOGGER.info("Building apk file...");
-        if (outApk.exists()) {
-            //noinspection ResultOfMethodCallIgnored
-            outApk.delete();
-        } else {
-            File outDir = outApk.getParentFile();
-            if (outDir != null && !outDir.exists()) {
-                //noinspection ResultOfMethodCallIgnored
-                outDir.mkdirs();
-            }
-        }
-        File assetDir = new File(mApkDir, "assets");
-        if (!assetDir.exists()) {
-            assetDir = null;
-        }
-        zipPackage(outApk, new File(mApkDir, APK_DIRNAME), assetDir);
-    }
-
-    private void zipPackage(File apkFile, File rawDir, File assetDir) throws AndrolibException {
-        try {
-            ZipUtils.zipFolders(rawDir, apkFile, assetDir, mApkInfo.doNotCompress);
-        } catch (IOException | BrutException ex) {
-            throw new AndrolibException(ex);
-        }
-    }
-
     private File[] getIncludeFiles() throws AndrolibException {
         UsesFramework usesFramework = mApkInfo.usesFramework;
         if (usesFramework == null) {

brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java

@@ -32,15 +32,18 @@ import org.apache.commons.io.FilenameUtils;
 
 import java.io.*;
 import java.util.*;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.logging.Logger;
 import java.util.regex.Pattern;
 
 public class ApkDecoder {
     private final static Logger LOGGER = Logger.getLogger(ApkDecoder.class.getName());
 
+    private final AtomicReference<RuntimeException> mBuildError = new AtomicReference<>(null);
     private final Config mConfig;
     private final ApkInfo mApkInfo;
-    private int mMinSdkVersion = 0;
+    private volatile int mMinSdkVersion = 0;
+    private BackgroundWorker mWorker;
 
     private final static String SMALI_DIRNAME = "smali";
     private final static String UNK_DIRNAME = "unknown";
@@ -75,6 +78,7 @@ public class ApkDecoder {
     public ApkInfo decode(File outDir) throws AndrolibException, IOException, DirectoryException {
         ExtFile apkFile = mApkInfo.getApkFile();
         try {
+            mWorker = new BackgroundWorker();
             if (!mConfig.forceDelete && outDir.exists()) {
                 throw new OutDirExistsException();
             }
@@ -93,6 +97,44 @@ public class ApkDecoder {
 
             LOGGER.info("Using Apktool " + ApktoolProperties.getVersion() + " on " + mApkInfo.apkFileName);
 
+            if (mApkInfo.hasSources()) {
+                switch (mConfig.decodeSources) {
+                    case Config.DECODE_SOURCES_NONE:
+                        copySourcesRaw(outDir, "classes.dex");
+                        break;
+                    case Config.DECODE_SOURCES_SMALI:
+                    case Config.DECODE_SOURCES_SMALI_ONLY_MAIN_CLASSES:
+                        scheduleDecodeSourcesSmali(outDir, "classes.dex");
+                        break;
+                }
+            }
+
+            if (mApkInfo.hasMultipleSources()) {
+                // foreach unknown dex file in root, lets disassemble it
+                Set<String> files = apkFile.getDirectory().getFiles(true);
+                for (String file : files) {
+                    if (file.endsWith(".dex")) {
+                        if (!file.equalsIgnoreCase("classes.dex")) {
+                            switch(mConfig.decodeSources) {
+                                case Config.DECODE_SOURCES_NONE:
+                                    copySourcesRaw(outDir, file);
+                                    break;
+                                case Config.DECODE_SOURCES_SMALI:
+                                    scheduleDecodeSourcesSmali(outDir, file);
+                                    break;
+                                case Config.DECODE_SOURCES_SMALI_ONLY_MAIN_CLASSES:
+                                    if (file.startsWith("classes") && file.endsWith(".dex")) {
+                                        scheduleDecodeSourcesSmali(outDir, file);
+                                    } else {
+                                        copySourcesRaw(outDir, file);
+                                    }
+                                    break;
+                            }
+                        }
+                    }
+                }
+            }
+
             ResourcesDecoder resourcesDecoder = new ResourcesDecoder(mConfig, mApkInfo);
 
             if (mApkInfo.hasResources()) {
@@ -117,42 +159,13 @@ public class ApkDecoder {
             }
             resourcesDecoder.updateApkInfo(outDir);
 
-            if (mApkInfo.hasSources()) {
+            copyRawFiles(outDir);
-                switch (mConfig.decodeSources) {
+            copyUnknownFiles(outDir);
-                    case Config.DECODE_SOURCES_NONE:
+            recordUncompressedFiles(resourcesDecoder.getResFileMapping());
-                        copySourcesRaw(outDir, "classes.dex");
+            copyOriginalFiles(outDir);
-                        break;
+            mWorker.waitForFinish();
-                    case Config.DECODE_SOURCES_SMALI:
+            if (mBuildError.get() != null) {
-                    case Config.DECODE_SOURCES_SMALI_ONLY_MAIN_CLASSES:
+                throw mBuildError.get();
-                        decodeSourcesSmali(outDir, "classes.dex");
-                        break;
-                }
-            }
-
-            if (mApkInfo.hasMultipleSources()) {
-                // foreach unknown dex file in root, lets disassemble it
-                Set<String> files = apkFile.getDirectory().getFiles(true);
-                for (String file : files) {
-                    if (file.endsWith(".dex")) {
-                        if (!file.equalsIgnoreCase("classes.dex")) {
-                            switch(mConfig.decodeSources) {
-                                case Config.DECODE_SOURCES_NONE:
-                                    copySourcesRaw(outDir, file);
-                                    break;
-                                case Config.DECODE_SOURCES_SMALI:
-                                    decodeSourcesSmali(outDir, file);
-                                    break;
-                                case Config.DECODE_SOURCES_SMALI_ONLY_MAIN_CLASSES:
-                                    if (file.startsWith("classes") && file.endsWith(".dex")) {
-                                        decodeSourcesSmali(outDir, file);
-                                    } else {
-                                        copySourcesRaw(outDir, file);
-                                    }
-                                    break;
-                            }
-                        }
-                    }
-                }
             }
 
             // In case we have no resources. We should store the minSdk we pulled from the source opcode api level
@@ -160,14 +173,11 @@ public class ApkDecoder {
                 mApkInfo.setSdkInfoField("minSdkVersion", Integer.toString(mMinSdkVersion));
             }
 
-            copyRawFiles(outDir);
-            copyUnknownFiles(outDir);
-            recordUncompressedFiles(resourcesDecoder.getResFileMapping());
-            copyOriginalFiles(outDir);
             writeApkInfo(outDir);
 
             return mApkInfo;
         } finally {
+            mWorker.shutdownNow();
             try {
                 apkFile.close();
             } catch (IOException ignored) {}
@@ -205,6 +215,17 @@ public class ApkDecoder {
         }
     }
 
+    private void scheduleDecodeSourcesSmali(File outDir, String filename) {
+        Runnable r = () -> {
+            try {
+                decodeSourcesSmali(outDir, filename);
+            } catch (AndrolibException e) {
+                mBuildError.compareAndSet(null, new RuntimeException(e));
+            }
+        };
+        mWorker.submit(r);
+    }
+
     private void decodeSourcesSmali(File outDir, String filename) throws AndrolibException {
         try {
             File smaliDir;

brut.apktool/apktool-lib/src/main/java/brut/androlib/BackgroundWorker.java

@@ -0,0 +1,65 @@
+package brut.androlib;
+
+import java.util.ArrayList;
+import java.util.concurrent.*;
+
+public class BackgroundWorker {
+
+    private static final int THREADS_COUNT = Runtime.getRuntime().availableProcessors();
+    private final ArrayList<Future<?>> mWorkerFutures = new ArrayList<>();
+    private final ExecutorService mExecutor;
+    private volatile boolean mSubmitAllowed = true;
+
+    public BackgroundWorker() {
+        this(THREADS_COUNT);
+    }
+
+    public BackgroundWorker(int threads) {
+        mExecutor = Executors.newFixedThreadPool(threads);
+    }
+
+    public void waitForFinish() {
+        checkState();
+        mSubmitAllowed = false;
+        for (Future<?> future : mWorkerFutures) {
+            try {
+                future.get();
+            } catch (InterruptedException | ExecutionException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        mWorkerFutures.clear();
+        mSubmitAllowed = true;
+    }
+
+    public void clearFutures() {
+        mWorkerFutures.clear();
+    }
+
+    private void checkState() {
+        if (!mSubmitAllowed) {
+            throw new IllegalStateException("BackgroundWorker is not ready");
+        }
+    }
+
+    public void shutdownNow() {
+        mSubmitAllowed = false;
+        mExecutor.shutdownNow();
+    }
+
+    public ExecutorService getExecutor() {
+        return mExecutor;
+    }
+
+    public void submit(Runnable task) {
+        checkState();
+        mWorkerFutures.add(mExecutor.submit(task));
+    }
+
+    public <T> Future<T> submit(Callable<T> task) {
+        checkState();
+        Future<T> future = mExecutor.submit(task);
+        mWorkerFutures.add(future);
+        return future;
+    }
+}

brut.j.dir/src/main/java/brut/directory/ZipUtils.java

@@ -39,15 +39,21 @@ public class ZipUtils {
     public static void zipFolders(final File folder, final File zip, final File assets, final Collection<String> doNotCompress)
             throws BrutException, IOException {
 
-        mDoNotCompress = doNotCompress;
         ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(zip.toPath()));
+        zipFoldersPreserveStream(folder, zipOutputStream, assets, doNotCompress);
+        zipOutputStream.close();
+    }
+
+    public static void zipFoldersPreserveStream(final File folder, final ZipOutputStream zipOutputStream, final File assets, final Collection<String> doNotCompress)
+            throws BrutException, IOException {
+
+        mDoNotCompress = doNotCompress;
         zipFolders(folder, zipOutputStream);
 
         // We manually set the assets because we need to retain the folder structure
         if (assets != null) {
             processFolder(assets, zipOutputStream, assets.getPath().length() - 6);
         }
-        zipOutputStream.close();
     }
 
     private static void zipFolders(final File folder, final ZipOutputStream outputStream)