From b995c05fd355c216ee7920a60fe3034bc6cdde2c Mon Sep 17 00:00:00 2001 From: LisoUseInAIKyrios <118716522+LisoUseInAIKyrios@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:47:06 +0200 Subject: [PATCH] perf: Buffer zip output writes and write decoded res files in parallel --- .../reandroid/apk/ApkModuleXmlDecoder.java | 122 ++++++++++++++++-- .../archive/io/FileChannelOutputStream.java | 48 ++++++- .../reandroid/archive/io/ZipFileOutput.java | 11 ++ 3 files changed, 162 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/reandroid/apk/ApkModuleXmlDecoder.java b/src/main/java/com/reandroid/apk/ApkModuleXmlDecoder.java index 256e9a546..2e0de5dc0 100644 --- a/src/main/java/com/reandroid/apk/ApkModuleXmlDecoder.java +++ b/src/main/java/com/reandroid/apk/ApkModuleXmlDecoder.java @@ -24,6 +24,7 @@ import com.reandroid.arsc.chunk.xml.ResXmlDocument; import com.reandroid.arsc.coder.xml.XmlCoder; import com.reandroid.arsc.list.OverlayableList; +import com.reandroid.utils.io.FileUtil; import com.reandroid.utils.io.IOUtil; import com.reandroid.arsc.value.*; import com.reandroid.json.JSONObject; @@ -31,9 +32,17 @@ import com.reandroid.xml.XmlIndentingSerializer; import org.xmlpull.v1.XmlSerializer; +import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.InterruptedIOException; +import java.io.OutputStream; import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; import java.util.function.Predicate; public class ApkModuleXmlDecoder extends ApkModuleDecoder implements Predicate { @@ -91,15 +100,22 @@ private void decodeResFiles(File mainDirectory) throws IOException{ logMessage("Res files: " + TableBlock.DIRECTORY_NAME); } List resFileList = getApkModule().listResFiles(); - for(ResFile resFile:resFileList){ - decodeResFile(mainDirectory, resFile); + ResFileWriter resFileWriter = new ResFileWriter(); + try{ + for(ResFile resFile:resFileList){ + decodeResFile(mainDirectory, resFile, resFileWriter); + } + }catch (IOException | RuntimeException exception){ + resFileWriter.abort(); + throw exception; } + resFileWriter.finish(); } - private void decodeResFile(File mainDirectory, ResFile resFile) + private void decodeResFile(File mainDirectory, ResFile resFile, ResFileWriter resFileWriter) throws IOException{ if(resFile.isBinaryXml()){ try{ - decodeResXml(mainDirectory, resFile); + decodeResXml(mainDirectory, resFile, resFileWriter); }catch (Exception ex){ logOrThrow("Failed to decode: " + resFile.getFilePath(), ex); @@ -111,9 +127,9 @@ private void decodeResFile(File mainDirectory, ResFile resFile) logMessage("Ignore non bin xml: " + path); return; } - decodeResRaw(mainDirectory, resFile); + decodeResRaw(mainDirectory, resFile, resFileWriter); } - private void decodeResRaw(File mainDirectory, ResFile resFile) + private void decodeResRaw(File mainDirectory, ResFile resFile, ResFileWriter resFileWriter) throws IOException { Entry entry = resFile.pickOne(); PackageBlock packageBlock = entry.getPackageBlock(); @@ -121,13 +137,13 @@ private void decodeResRaw(File mainDirectory, ResFile resFile) File file = toDecodeResFile(mainDirectory, resFile, packageBlock); InputSource inputSource = resFile.getInputSource(); logVerbose(inputSource.getAlias()); - inputSource.write(file); + resFileWriter.write(file, IOUtil.readFully(inputSource.openStream())); if(!keepResPath()){ addDecodedEntry(entry); } addDecodedPath(inputSource.getAlias()); } - private void decodeResXml(File mainDirectory, ResFile resFile) + private void decodeResXml(File mainDirectory, ResFile resFile, ResFileWriter resFileWriter) throws IOException{ Entry entry = resFile.pickOne(); PackageBlock packageBlock = entry.getPackageBlock(); @@ -136,7 +152,9 @@ private void decodeResXml(File mainDirectory, ResFile resFile) InputSource inputSource = resFile.getInputSource(); logVerbose(inputSource.getAlias()); - serializeXml(packageBlock, resFile.getInputSource(), file); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + serializeXml(packageBlock, inputSource, outputStream); + resFileWriter.write(file, outputStream.toByteArray()); if(!keepResPath()){ addDecodedEntry(entry); @@ -247,19 +265,23 @@ private void decodeAndroidManifestBin(File mainDirectory) } private void serializeXml(PackageBlock packageBlock, ResXmlDocument document, File outFile) throws IOException { + serializeXml(packageBlock, document, FileUtil.outputStream(outFile)); + } + private void serializeXml(PackageBlock packageBlock, ResXmlDocument document, OutputStream outputStream) + throws IOException { if(packageBlock != null && document.getPackageBlock() == null){ document.setPackageBlock(packageBlock); } - XmlSerializer serializer = XMLFactory.newSerializer(outFile, document.getEncoding()); + XmlSerializer serializer = XMLFactory.newSerializer(outputStream, document.getEncoding()); document.serialize(serializer); IOUtil.close(serializer); } - private void serializeXml(PackageBlock packageBlock, InputSource inputSource, File outFile) + private void serializeXml(PackageBlock packageBlock, InputSource inputSource, OutputStream outputStream) throws IOException { ResXmlDocument document = new ResXmlDocument(); document.readBytes(inputSource.openStream()); document.setPackageBlock(packageBlock); - serializeXml(packageBlock, document, outFile); + serializeXml(packageBlock, document, outputStream); } private void addDecodedEntry(Entry entry){ if(entry.isNull()){ @@ -308,4 +330,80 @@ private void decodeOverlayable(File mainDirectory, PackageBlock packageBlock) th public boolean test(Entry entry) { return containsDecodedEntry(entry); } + + /** + * Writes decoded res files on background threads, as creating many small files is mostly + * time spent waiting on the file system. The contents are decoded by the caller because + * the table is not thread safe, so only the file writing is done in parallel. + */ + private class ResFileWriter { + private static final int RES_FILE_WRITE_THREADS = 2; // TODO: Use number of device cores? + private static final int MAX_PENDING_WRITE_BYTES = 16 * 1024 * 1024; + + private final ExecutorService executor; + private final Semaphore pendingBytes; + private final Set createdDirectories; + private volatile File failedFile; + private volatile IOException failure; + + ResFileWriter(){ + this.executor = Executors.newFixedThreadPool(RES_FILE_WRITE_THREADS, runnable -> { + Thread thread = new Thread(runnable, "res-file-writer"); + thread.setDaemon(true); + return thread; + }); + this.pendingBytes = new Semaphore(MAX_PENDING_WRITE_BYTES); + this.createdDirectories = new HashSet<>(); + } + void write(File file, byte[] bytes) throws IOException { + if(failure != null){ + return; + } + File dir = file.getParentFile(); + if(dir != null && createdDirectories.add(dir) && !dir.exists()){ + dir.mkdirs(); + } + // Limits the memory used by contents waiting to be written. + int permits = Math.min(Math.max(bytes.length, 1), MAX_PENDING_WRITE_BYTES); + try{ + pendingBytes.acquire(permits); + }catch (InterruptedException exception){ + Thread.currentThread().interrupt(); + throw new InterruptedIOException("Interrupted writing: " + file); + } + executor.execute(() -> { + try(FileOutputStream outputStream = new FileOutputStream(file)){ + outputStream.write(bytes); + }catch (IOException exception){ + synchronized (this){ + if(failure == null){ + failedFile = file; + failure = exception; + } + } + }finally { + pendingBytes.release(permits); + } + }); + } + void finish() throws IOException { + executor.shutdown(); + try{ + while (!executor.awaitTermination(1, TimeUnit.MINUTES)){ + logVerbose("Waiting for res files to be written ..."); + } + }catch (InterruptedException exception){ + executor.shutdownNow(); + Thread.currentThread().interrupt(); + throw new InterruptedIOException("Interrupted writing res files"); + } + IOException failure = this.failure; + if(failure != null){ + logOrThrow("Failed to write: " + failedFile, failure); + } + } + void abort(){ + executor.shutdownNow(); + } + } } diff --git a/src/main/java/com/reandroid/archive/io/FileChannelOutputStream.java b/src/main/java/com/reandroid/archive/io/FileChannelOutputStream.java index 22b4a7b9c..1ff60bdee 100644 --- a/src/main/java/com/reandroid/archive/io/FileChannelOutputStream.java +++ b/src/main/java/com/reandroid/archive/io/FileChannelOutputStream.java @@ -22,8 +22,11 @@ public class FileChannelOutputStream extends OutputStream { private final FileChannel fileChannel; + private final ByteBuffer buffer; + public FileChannelOutputStream(FileChannel fileChannel){ this.fileChannel = fileChannel; + this.buffer = ByteBuffer.allocate(BUFFER_SIZE); } @Override public void write(byte[] bytes) throws IOException { @@ -31,17 +34,48 @@ public void write(byte[] bytes) throws IOException { } @Override public void write(byte[] bytes, int offset, int length) throws IOException { - long position = fileChannel.position(); - length = fileChannel.write(ByteBuffer.wrap(bytes, offset, length)); - fileChannel.position(position + length); + ByteBuffer buffer = this.buffer; + if(length > buffer.remaining()){ + flush(); + if(length >= buffer.capacity()){ + writeFully(ByteBuffer.wrap(bytes, offset, length)); + return; + } + } + buffer.put(bytes, offset, length); } @Override public void write(int i) throws IOException { - byte b = (byte) (i & 0xff); - write(new byte[]{b}); + ByteBuffer buffer = this.buffer; + if(!buffer.hasRemaining()){ + flush(); + } + buffer.put((byte) i); } + /** + * Writes the buffered bytes to the channel at its current position. + * Must be called before the channel is used directly. + */ @Override - public void close(){ - + public void flush() throws IOException { + ByteBuffer buffer = this.buffer; + if(buffer.position() == 0){ + return; + } + buffer.flip(); + writeFully(buffer); + buffer.clear(); + } + private void writeFully(ByteBuffer byteBuffer) throws IOException { + FileChannel fileChannel = this.fileChannel; + while (byteBuffer.hasRemaining()){ + fileChannel.write(byteBuffer); + } } + @Override + public void close() throws IOException { + flush(); + } + + private static final int BUFFER_SIZE = 64 * 1024; } diff --git a/src/main/java/com/reandroid/archive/io/ZipFileOutput.java b/src/main/java/com/reandroid/archive/io/ZipFileOutput.java index ea2c89aab..ba422ccbe 100644 --- a/src/main/java/com/reandroid/archive/io/ZipFileOutput.java +++ b/src/main/java/com/reandroid/archive/io/ZipFileOutput.java @@ -35,6 +35,7 @@ public File getFile() { return file; } public void write(FileChannel input, long length) throws IOException{ + flushOutputStream(); FileChannel fileChannel = getFileChannel(); long pos = fileChannel.position(); @@ -52,16 +53,19 @@ public void write(FileChannel input, long length) throws IOException{ @Override public long position() throws IOException { + flushOutputStream(); return getFileChannel().position(); } @Override public void position(long pos) throws IOException { + flushOutputStream(); getFileChannel().position(pos); } @Override public void close() throws IOException { FileChannel fileChannel = this.fileChannel; if(fileChannel != null){ + flushOutputStream(); fileChannel.close(); } } @@ -87,6 +91,7 @@ private FileChannel getFileChannel() throws IOException { @Override public void write(InputStream inputStream) throws IOException { + flushOutputStream(); FileChannel fileChannel = getFileChannel(); long pos = fileChannel.position(); int bufferLength = 1024 * 1000 * 10; @@ -111,6 +116,12 @@ public FileChannelOutputStream getOutputStream() throws IOException { return outputStream; } + private void flushOutputStream() throws IOException { + FileChannelOutputStream outputStream = this.outputStream; + if(outputStream != null){ + outputStream.flush(); + } + } private static void initFile(File file) throws IOException{ if(file.isDirectory()){