From 81aae6936a4d0228e9b10aaeffaaaeda04eaf81c Mon Sep 17 00:00:00 2001 From: Cregrant <62436046+Cregrant@users.noreply.github.com> Date: Tue, 26 Dec 2023 18:20:26 +0700 Subject: [PATCH] Feature: Parallel Building (#3476) * perf: process smali code in parallel Note: backsmali can't be properly multithreaded because of the synchronized methods inside * perf: start backsmali concurrently with a resources decompiler * perf: speed up apk building by skipping temp archive creation Now we're not compressing the same data twice * refactor: extract duplicated code * refactor: rename methods and inline some comments --- .../main/java/brut/androlib/ApkBuilder.java | 213 ++++++++---------- .../main/java/brut/androlib/ApkDecoder.java | 103 +++++---- .../java/brut/androlib/BackgroundWorker.java | 65 ++++++ .../main/java/brut/directory/ZipUtils.java | 10 +- 4 files changed, 227 insertions(+), 164 deletions(-) create mode 100644 brut.apktool/apktool-lib/src/main/java/brut/androlib/BackgroundWorker.java diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java index 39eecdb6..50276514 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java @@ -42,17 +42,19 @@ import javax.xml.transform.TransformerException; import java.io.*; import java.nio.file.Files; import java.util.*; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Logger; import java.util.zip.CRC32; import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; public class ApkBuilder { private final static Logger LOGGER = Logger.getLogger(ApkBuilder.class.getName()); + private final AtomicReference mBuildError = new AtomicReference<>(null); private final Config mConfig; private final ExtFile mApkDir; + private BackgroundWorker mWorker; private ApkInfo mApkInfo; private int mMinSdkVersion = 0; @@ -78,51 +80,55 @@ public class ApkBuilder { public void build(File outFile) throws BrutException { LOGGER.info("Using Apktool " + ApktoolProperties.getVersion()); + try { + mWorker = new BackgroundWorker(); + mApkInfo = ApkInfo.load(mApkDir); - mApkInfo = ApkInfo.load(mApkDir); - - if (mApkInfo.getSdkInfo() != null && mApkInfo.getSdkInfo().get("minSdkVersion") != null) { - String minSdkVersion = mApkInfo.getSdkInfo().get("minSdkVersion"); - mMinSdkVersion = mApkInfo.getMinSdkVersionFromAndroidCodename(minSdkVersion); - } - - if (outFile == null) { - String outFileName = mApkInfo.apkFileName; - outFile = new File(mApkDir, "dist" + File.separator + (outFileName == null ? "out.apk" : outFileName)); - } - - //noinspection ResultOfMethodCallIgnored - new File(mApkDir, APK_DIRNAME).mkdirs(); - File manifest = new File(mApkDir, "AndroidManifest.xml"); - File manifestOriginal = new File(mApkDir, "AndroidManifest.xml.orig"); - - buildSources(); - buildNonDefaultSources(); - buildManifestFile(manifest, manifestOriginal); - buildResources(); - buildLibs(); - buildCopyOriginalFiles(); - buildApk(outFile); - - // we must go after the Apk is built, and copy the files in via Zip - // this is because Aapt won't add files it doesn't know (ex unknown files) - buildUnknownFiles(outFile); - - // we copied the AndroidManifest.xml to AndroidManifest.xml.orig so we can edit it - // lets restore the unedited one, to not change the original - if (manifest.isFile() && manifest.exists() && manifestOriginal.isFile()) { - try { - if (new File(mApkDir, "AndroidManifest.xml").delete()) { - FileUtils.moveFile(manifestOriginal, manifest); - } - } catch (IOException ex) { - throw new AndrolibException(ex.getMessage()); + if (mApkInfo.getSdkInfo() != null && mApkInfo.getSdkInfo().get("minSdkVersion") != null) { + String minSdkVersion = mApkInfo.getSdkInfo().get("minSdkVersion"); + mMinSdkVersion = mApkInfo.getMinSdkVersionFromAndroidCodename(minSdkVersion); } + + if (outFile == null) { + String outFileName = mApkInfo.apkFileName; + outFile = new File(mApkDir, "dist" + File.separator + (outFileName == null ? "out.apk" : outFileName)); + } + + //noinspection ResultOfMethodCallIgnored + new File(mApkDir, APK_DIRNAME).mkdirs(); + File manifest = new File(mApkDir, "AndroidManifest.xml"); + File manifestOriginal = new File(mApkDir, "AndroidManifest.xml.orig"); + + scheduleBuildDexFiles(); + backupManifestFile(manifest, manifestOriginal); + buildResources(); + copyLibs(); + copyOriginalFilesIfEnabled(); + mWorker.waitForFinish(); + if (mBuildError.get() != null) { + throw mBuildError.get(); + } + + buildApk(outFile); + + // we copied the AndroidManifest.xml to AndroidManifest.xml.orig so we can edit it + // lets restore the unedited one, to not change the original + if (manifest.isFile() && manifest.exists() && manifestOriginal.isFile()) { + try { + if (new File(mApkDir, "AndroidManifest.xml").delete()) { + FileUtils.moveFile(manifestOriginal, manifest); + } + } catch (IOException ex) { + throw new AndrolibException(ex.getMessage()); + } + } + LOGGER.info("Built apk into: " + outFile.getPath()); + } finally { + mWorker.shutdownNow(); } - LOGGER.info("Built apk into: " + outFile.getPath()); } - private void buildManifestFile(File manifest, File manifestOriginal) throws AndrolibException { + private void backupManifestFile(File manifest, File manifestOriginal) throws AndrolibException { // If we decoded in "raw", we cannot patch AndroidManifest if (new File(mApkDir, "resources.arsc").exists()) { return; @@ -141,24 +147,17 @@ public class ApkBuilder { } } - private void buildSources() throws AndrolibException { - if (!buildSourcesRaw("classes.dex") && !buildSourcesSmali("smali", "classes.dex")) { - LOGGER.warning("Could not find sources"); - } - } - - private void buildNonDefaultSources() throws AndrolibException { + private void scheduleBuildDexFiles() throws AndrolibException { try { + mWorker.submit(() -> scheduleDexBuild("classes.dex", "smali")); + // loop through any smali_ directories for multi-dex apks Map dirs = mApkDir.getDirectory().getDirs(); for (Map.Entry directory : dirs.entrySet()) { String name = directory.getKey(); if (name.startsWith("smali_")) { String filename = name.substring(name.indexOf("_") + 1) + ".dex"; - - if (!buildSourcesRaw(filename) && !buildSourcesSmali(name, filename)) { - LOGGER.warning("Could not find sources"); - } + mWorker.submit(() -> scheduleDexBuild(filename, name)); } } @@ -177,6 +176,19 @@ public class ApkBuilder { } } + private void scheduleDexBuild(String filename, String smali) { + try { + if (mBuildError.get() != null) { + return; + } + if (!buildSourcesRaw(filename) && !buildSourcesSmali(smali, filename)) { + LOGGER.warning("Could not find sources"); + } + } catch (AndrolibException e) { + mBuildError.compareAndSet(null, e); + } + } + private boolean buildSourcesRaw(String filename) throws AndrolibException { File working = new File(mApkDir, filename); if (!working.exists()) { @@ -214,6 +226,7 @@ public class ApkBuilder { } private void buildResources() throws BrutException { + // create res folder, manifest file and resources.arsc if (!buildResourcesRaw() && !buildResourcesFull() && !buildManifest()) { LOGGER.warning("Could not find resources"); } @@ -375,7 +388,7 @@ public class ApkBuilder { } } - private void buildLibs() throws AndrolibException { + private void copyLibs() throws AndrolibException { buildLibrary("lib"); buildLibrary("libs"); buildLibrary("kotlin"); @@ -401,7 +414,7 @@ public class ApkBuilder { } } - private void buildCopyOriginalFiles() throws AndrolibException { + private void copyOriginalFilesIfEnabled() throws AndrolibException { if (mConfig.copyOriginalFiles) { File originalDir = new File(mApkDir, "original"); if (originalDir.exists()) { @@ -427,49 +440,34 @@ public class ApkBuilder { } } - private void buildUnknownFiles(File outFile) throws AndrolibException { - if (mApkInfo.unknownFiles != null) { - LOGGER.info("Copying unknown files/dir..."); - - Map files = mApkInfo.unknownFiles; - File tempFile = new File(outFile.getParent(), outFile.getName() + ".apktool_temp"); - boolean renamed = outFile.renameTo(tempFile); - if (!renamed) { - throw new AndrolibException("Unable to rename temporary file"); - } - - try ( - ZipFile inputFile = new ZipFile(tempFile); - ZipOutputStream actualOutput = new ZipOutputStream(Files.newOutputStream(outFile.toPath())) - ) { - copyExistingFiles(inputFile, actualOutput); - copyUnknownFiles(actualOutput, files); - } catch (IOException | BrutException ex) { - throw new AndrolibException(ex); - } - - // Remove our temporary file. + private void buildApk(File outApk) throws AndrolibException { + LOGGER.info("Building apk file..."); + if (outApk.exists()) { //noinspection ResultOfMethodCallIgnored - tempFile.delete(); - } - } - - private void copyExistingFiles(ZipFile inputFile, ZipOutputStream outputFile) throws IOException { - // First, copy the contents from the existing outFile: - Enumeration entries = inputFile.entries(); - while (entries.hasMoreElements()) { - ZipEntry entry = new ZipEntry(entries.nextElement()); - - // We can't reuse the compressed size because it depends on compression sizes. - entry.setCompressedSize(-1); - outputFile.putNextEntry(entry); - - // No need to create directory entries in the final apk - if (!entry.isDirectory()) { - BrutIO.copy(inputFile, outputFile, entry); + outApk.delete(); + } else { + File outDir = outApk.getParentFile(); + if (outDir != null && !outDir.exists()) { + //noinspection ResultOfMethodCallIgnored + outDir.mkdirs(); } + } + File assetDir = new File(mApkDir, "assets"); + if (!assetDir.exists()) { + assetDir = null; + } + try (ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(outApk.toPath()))) { + // zip all AAPT-generated files + ZipUtils.zipFoldersPreserveStream(new File(mApkDir, APK_DIRNAME), zipOutputStream, assetDir, mApkInfo.doNotCompress); - outputFile.closeEntry(); + // we must copy some files manually + // this is because Aapt won't add files it doesn't know (ex unknown files) + if (mApkInfo.unknownFiles != null) { + LOGGER.info("Copying unknown files/dir..."); + copyUnknownFiles(zipOutputStream, mApkInfo.unknownFiles); + } + } catch (IOException | BrutException e) { + throw new AndrolibException(e); } } @@ -513,33 +511,6 @@ public class ApkBuilder { } } - private void buildApk(File outApk) throws AndrolibException { - LOGGER.info("Building apk file..."); - if (outApk.exists()) { - //noinspection ResultOfMethodCallIgnored - outApk.delete(); - } else { - File outDir = outApk.getParentFile(); - if (outDir != null && !outDir.exists()) { - //noinspection ResultOfMethodCallIgnored - outDir.mkdirs(); - } - } - File assetDir = new File(mApkDir, "assets"); - if (!assetDir.exists()) { - assetDir = null; - } - zipPackage(outApk, new File(mApkDir, APK_DIRNAME), assetDir); - } - - private void zipPackage(File apkFile, File rawDir, File assetDir) throws AndrolibException { - try { - ZipUtils.zipFolders(rawDir, apkFile, assetDir, mApkInfo.doNotCompress); - } catch (IOException | BrutException ex) { - throw new AndrolibException(ex); - } - } - private File[] getIncludeFiles() throws AndrolibException { UsesFramework usesFramework = mApkInfo.usesFramework; if (usesFramework == null) { diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java index 1d04985b..ec711354 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkDecoder.java @@ -32,15 +32,18 @@ import org.apache.commons.io.FilenameUtils; import java.io.*; import java.util.*; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Logger; import java.util.regex.Pattern; public class ApkDecoder { private final static Logger LOGGER = Logger.getLogger(ApkDecoder.class.getName()); + private final AtomicReference mBuildError = new AtomicReference<>(null); private final Config mConfig; private final ApkInfo mApkInfo; - private int mMinSdkVersion = 0; + private volatile int mMinSdkVersion = 0; + private BackgroundWorker mWorker; private final static String SMALI_DIRNAME = "smali"; private final static String UNK_DIRNAME = "unknown"; @@ -75,6 +78,7 @@ public class ApkDecoder { public ApkInfo decode(File outDir) throws AndrolibException, IOException, DirectoryException { ExtFile apkFile = mApkInfo.getApkFile(); try { + mWorker = new BackgroundWorker(); if (!mConfig.forceDelete && outDir.exists()) { throw new OutDirExistsException(); } @@ -93,6 +97,44 @@ public class ApkDecoder { LOGGER.info("Using Apktool " + ApktoolProperties.getVersion() + " on " + mApkInfo.apkFileName); + if (mApkInfo.hasSources()) { + switch (mConfig.decodeSources) { + case Config.DECODE_SOURCES_NONE: + copySourcesRaw(outDir, "classes.dex"); + break; + case Config.DECODE_SOURCES_SMALI: + case Config.DECODE_SOURCES_SMALI_ONLY_MAIN_CLASSES: + scheduleDecodeSourcesSmali(outDir, "classes.dex"); + break; + } + } + + if (mApkInfo.hasMultipleSources()) { + // foreach unknown dex file in root, lets disassemble it + Set files = apkFile.getDirectory().getFiles(true); + for (String file : files) { + if (file.endsWith(".dex")) { + if (!file.equalsIgnoreCase("classes.dex")) { + switch(mConfig.decodeSources) { + case Config.DECODE_SOURCES_NONE: + copySourcesRaw(outDir, file); + break; + case Config.DECODE_SOURCES_SMALI: + scheduleDecodeSourcesSmali(outDir, file); + break; + case Config.DECODE_SOURCES_SMALI_ONLY_MAIN_CLASSES: + if (file.startsWith("classes") && file.endsWith(".dex")) { + scheduleDecodeSourcesSmali(outDir, file); + } else { + copySourcesRaw(outDir, file); + } + break; + } + } + } + } + } + ResourcesDecoder resourcesDecoder = new ResourcesDecoder(mConfig, mApkInfo); if (mApkInfo.hasResources()) { @@ -117,42 +159,13 @@ public class ApkDecoder { } resourcesDecoder.updateApkInfo(outDir); - if (mApkInfo.hasSources()) { - switch (mConfig.decodeSources) { - case Config.DECODE_SOURCES_NONE: - copySourcesRaw(outDir, "classes.dex"); - break; - case Config.DECODE_SOURCES_SMALI: - case Config.DECODE_SOURCES_SMALI_ONLY_MAIN_CLASSES: - decodeSourcesSmali(outDir, "classes.dex"); - break; - } - } - - if (mApkInfo.hasMultipleSources()) { - // foreach unknown dex file in root, lets disassemble it - Set files = apkFile.getDirectory().getFiles(true); - for (String file : files) { - if (file.endsWith(".dex")) { - if (!file.equalsIgnoreCase("classes.dex")) { - switch(mConfig.decodeSources) { - case Config.DECODE_SOURCES_NONE: - copySourcesRaw(outDir, file); - break; - case Config.DECODE_SOURCES_SMALI: - decodeSourcesSmali(outDir, file); - break; - case Config.DECODE_SOURCES_SMALI_ONLY_MAIN_CLASSES: - if (file.startsWith("classes") && file.endsWith(".dex")) { - decodeSourcesSmali(outDir, file); - } else { - copySourcesRaw(outDir, file); - } - break; - } - } - } - } + copyRawFiles(outDir); + copyUnknownFiles(outDir); + recordUncompressedFiles(resourcesDecoder.getResFileMapping()); + copyOriginalFiles(outDir); + mWorker.waitForFinish(); + if (mBuildError.get() != null) { + throw mBuildError.get(); } // In case we have no resources. We should store the minSdk we pulled from the source opcode api level @@ -160,14 +173,11 @@ public class ApkDecoder { mApkInfo.setSdkInfoField("minSdkVersion", Integer.toString(mMinSdkVersion)); } - copyRawFiles(outDir); - copyUnknownFiles(outDir); - recordUncompressedFiles(resourcesDecoder.getResFileMapping()); - copyOriginalFiles(outDir); writeApkInfo(outDir); return mApkInfo; } finally { + mWorker.shutdownNow(); try { apkFile.close(); } catch (IOException ignored) {} @@ -205,6 +215,17 @@ public class ApkDecoder { } } + private void scheduleDecodeSourcesSmali(File outDir, String filename) { + Runnable r = () -> { + try { + decodeSourcesSmali(outDir, filename); + } catch (AndrolibException e) { + mBuildError.compareAndSet(null, new RuntimeException(e)); + } + }; + mWorker.submit(r); + } + private void decodeSourcesSmali(File outDir, String filename) throws AndrolibException { try { File smaliDir; diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/BackgroundWorker.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/BackgroundWorker.java new file mode 100644 index 00000000..5ba74fc9 --- /dev/null +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/BackgroundWorker.java @@ -0,0 +1,65 @@ +package brut.androlib; + +import java.util.ArrayList; +import java.util.concurrent.*; + +public class BackgroundWorker { + + private static final int THREADS_COUNT = Runtime.getRuntime().availableProcessors(); + private final ArrayList> mWorkerFutures = new ArrayList<>(); + private final ExecutorService mExecutor; + private volatile boolean mSubmitAllowed = true; + + public BackgroundWorker() { + this(THREADS_COUNT); + } + + public BackgroundWorker(int threads) { + mExecutor = Executors.newFixedThreadPool(threads); + } + + public void waitForFinish() { + checkState(); + mSubmitAllowed = false; + for (Future future : mWorkerFutures) { + try { + future.get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + mWorkerFutures.clear(); + mSubmitAllowed = true; + } + + public void clearFutures() { + mWorkerFutures.clear(); + } + + private void checkState() { + if (!mSubmitAllowed) { + throw new IllegalStateException("BackgroundWorker is not ready"); + } + } + + public void shutdownNow() { + mSubmitAllowed = false; + mExecutor.shutdownNow(); + } + + public ExecutorService getExecutor() { + return mExecutor; + } + + public void submit(Runnable task) { + checkState(); + mWorkerFutures.add(mExecutor.submit(task)); + } + + public Future submit(Callable task) { + checkState(); + Future future = mExecutor.submit(task); + mWorkerFutures.add(future); + return future; + } +} diff --git a/brut.j.dir/src/main/java/brut/directory/ZipUtils.java b/brut.j.dir/src/main/java/brut/directory/ZipUtils.java index 3e719900..361b9fb1 100644 --- a/brut.j.dir/src/main/java/brut/directory/ZipUtils.java +++ b/brut.j.dir/src/main/java/brut/directory/ZipUtils.java @@ -39,15 +39,21 @@ public class ZipUtils { public static void zipFolders(final File folder, final File zip, final File assets, final Collection doNotCompress) throws BrutException, IOException { - mDoNotCompress = doNotCompress; ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(zip.toPath())); + zipFoldersPreserveStream(folder, zipOutputStream, assets, doNotCompress); + zipOutputStream.close(); + } + + public static void zipFoldersPreserveStream(final File folder, final ZipOutputStream zipOutputStream, final File assets, final Collection doNotCompress) + throws BrutException, IOException { + + mDoNotCompress = doNotCompress; zipFolders(folder, zipOutputStream); // We manually set the assets because we need to retain the folder structure if (assets != null) { processFolder(assets, zipOutputStream, assets.getPath().length() - 6); } - zipOutputStream.close(); } private static void zipFolders(final File folder, final ZipOutputStream outputStream)