Skip to content

perf: Buffer zip output writes and write decoded res files in parallel #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
LisoUseInAIKyrios merged 1 commit into from
Oct 3, 2026
Merged

perf: Buffer zip output writes and write decoded res files in parallel #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 110 additions & 12 deletions src/main/java/com/reandroid/apk/ApkModuleXmlDecoder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,25 @@
import com.reandroid.arsc.chunk.xml.ResXmlDocument;
import com.reandroid.arsc.coder.xml.XmlCoder;
import com.reandroid.arsc.list.OverlayableList;
import com.reandroid.utils.io.FileUtil;
import com.reandroid.utils.io.IOUtil;
import com.reandroid.arsc.value.*;
import com.reandroid.json.JSONObject;
import com.reandroid.xml.XMLFactory;
import com.reandroid.xml.XmlIndentingSerializer;
import org.xmlpull.v1.XmlSerializer;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.OutputStream;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;

public class ApkModuleXmlDecoder extends ApkModuleDecoder implements Predicate<Entry> {
Expand Down Expand Up @@ -91,15 +100,22 @@ private void decodeResFiles(File mainDirectory) throws IOException{
logMessage("Res files: " + TableBlock.DIRECTORY_NAME);
}
List<ResFile> resFileList = getApkModule().listResFiles();
for(ResFile resFile:resFileList){
decodeResFile(mainDirectory, resFile);
ResFileWriter resFileWriter = new ResFileWriter();
try{
for(ResFile resFile:resFileList){
decodeResFile(mainDirectory, resFile, resFileWriter);
}
}catch (IOException | RuntimeException exception){
resFileWriter.abort();
throw exception;
}
resFileWriter.finish();
}
private void decodeResFile(File mainDirectory, ResFile resFile)
private void decodeResFile(File mainDirectory, ResFile resFile, ResFileWriter resFileWriter)
throws IOException{
if(resFile.isBinaryXml()){
try{
decodeResXml(mainDirectory, resFile);
decodeResXml(mainDirectory, resFile, resFileWriter);
}catch (Exception ex){
logOrThrow("Failed to decode: "
+ resFile.getFilePath(), ex);
Expand All @@ -111,23 +127,23 @@ private void decodeResFile(File mainDirectory, ResFile resFile)
logMessage("Ignore non bin xml: " + path);
return;
}
decodeResRaw(mainDirectory, resFile);
decodeResRaw(mainDirectory, resFile, resFileWriter);
}
private void decodeResRaw(File mainDirectory, ResFile resFile)
private void decodeResRaw(File mainDirectory, ResFile resFile, ResFileWriter resFileWriter)
throws IOException {
Entry entry = resFile.pickOne();
PackageBlock packageBlock = entry.getPackageBlock();

File file = toDecodeResFile(mainDirectory, resFile, packageBlock);
InputSource inputSource = resFile.getInputSource();
logVerbose(inputSource.getAlias());
inputSource.write(file);
resFileWriter.write(file, IOUtil.readFully(inputSource.openStream()));
if(!keepResPath()){
addDecodedEntry(entry);
}
addDecodedPath(inputSource.getAlias());
}
private void decodeResXml(File mainDirectory, ResFile resFile)
private void decodeResXml(File mainDirectory, ResFile resFile, ResFileWriter resFileWriter)
throws IOException{
Entry entry = resFile.pickOne();
PackageBlock packageBlock = entry.getPackageBlock();
Expand All @@ -136,7 +152,9 @@ private void decodeResXml(File mainDirectory, ResFile resFile)
InputSource inputSource = resFile.getInputSource();

logVerbose(inputSource.getAlias());
serializeXml(packageBlock, resFile.getInputSource(), file);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
serializeXml(packageBlock, inputSource, outputStream);
resFileWriter.write(file, outputStream.toByteArray());

if(!keepResPath()){
addDecodedEntry(entry);
Expand Down Expand Up @@ -247,19 +265,23 @@ private void decodeAndroidManifestBin(File mainDirectory)
}
private void serializeXml(PackageBlock packageBlock, ResXmlDocument document, File outFile)
throws IOException {
serializeXml(packageBlock, document, FileUtil.outputStream(outFile));
}
private void serializeXml(PackageBlock packageBlock, ResXmlDocument document, OutputStream outputStream)
throws IOException {
if(packageBlock != null && document.getPackageBlock() == null){
document.setPackageBlock(packageBlock);
}
XmlSerializer serializer = XMLFactory.newSerializer(outFile, document.getEncoding());
XmlSerializer serializer = XMLFactory.newSerializer(outputStream, document.getEncoding());
document.serialize(serializer);
IOUtil.close(serializer);
}
private void serializeXml(PackageBlock packageBlock, InputSource inputSource, File outFile)
private void serializeXml(PackageBlock packageBlock, InputSource inputSource, OutputStream outputStream)
throws IOException {
ResXmlDocument document = new ResXmlDocument();
document.readBytes(inputSource.openStream());
document.setPackageBlock(packageBlock);
serializeXml(packageBlock, document, outFile);
serializeXml(packageBlock, document, outputStream);
}
private void addDecodedEntry(Entry entry){
if(entry.isNull()){
Expand Down Expand Up @@ -308,4 +330,80 @@ private void decodeOverlayable(File mainDirectory, PackageBlock packageBlock) th
public boolean test(Entry entry) {
return containsDecodedEntry(entry);
}

/**
* Writes decoded res files on background threads, as creating many small files is mostly
* time spent waiting on the file system. The contents are decoded by the caller because
* the table is not thread safe, so only the file writing is done in parallel.
*/
private class ResFileWriter {
private static final int RES_FILE_WRITE_THREADS = 2; // TODO: Use number of device cores?
private static final int MAX_PENDING_WRITE_BYTES = 16 * 1024 * 1024;

private final ExecutorService executor;
private final Semaphore pendingBytes;
private final Set<File> createdDirectories;
private volatile File failedFile;
private volatile IOException failure;

ResFileWriter(){
this.executor = Executors.newFixedThreadPool(RES_FILE_WRITE_THREADS, runnable -> {
Thread thread = new Thread(runnable, "res-file-writer");
thread.setDaemon(true);
return thread;
});
this.pendingBytes = new Semaphore(MAX_PENDING_WRITE_BYTES);
this.createdDirectories = new HashSet<>();
}
void write(File file, byte[] bytes) throws IOException {
if(failure != null){
return;
}
File dir = file.getParentFile();
if(dir != null && createdDirectories.add(dir) && !dir.exists()){
dir.mkdirs();
}
// Limits the memory used by contents waiting to be written.
int permits = Math.min(Math.max(bytes.length, 1), MAX_PENDING_WRITE_BYTES);
try{
pendingBytes.acquire(permits);
}catch (InterruptedException exception){
Thread.currentThread().interrupt();
throw new InterruptedIOException("Interrupted writing: " + file);
}
executor.execute(() -> {
try(FileOutputStream outputStream = new FileOutputStream(file)){
outputStream.write(bytes);
}catch (IOException exception){
synchronized (this){
if(failure == null){
failedFile = file;
failure = exception;
}
}
}finally {
pendingBytes.release(permits);
}
});
}
void finish() throws IOException {
executor.shutdown();
try{
while (!executor.awaitTermination(1, TimeUnit.MINUTES)){
logVerbose("Waiting for res files to be written ...");
}
}catch (InterruptedException exception){
executor.shutdownNow();
Thread.currentThread().interrupt();
throw new InterruptedIOException("Interrupted writing res files");
}
IOException failure = this.failure;
if(failure != null){
logOrThrow("Failed to write: " + failedFile, failure);
}
}
void abort(){
executor.shutdownNow();
}
}
}
48 changes: 41 additions & 7 deletions src/main/java/com/reandroid/archive/io/FileChannelOutputStream.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,26 +22,60 @@

public class FileChannelOutputStream extends OutputStream {
private final FileChannel fileChannel;
private final ByteBuffer buffer;

public FileChannelOutputStream(FileChannel fileChannel){
this.fileChannel = fileChannel;
this.buffer = ByteBuffer.allocate(BUFFER_SIZE);
}
@Override
public void write(byte[] bytes) throws IOException {
write(bytes, 0, bytes.length);
}
@Override
public void write(byte[] bytes, int offset, int length) throws IOException {
long position = fileChannel.position();
length = fileChannel.write(ByteBuffer.wrap(bytes, offset, length));
fileChannel.position(position + length);
ByteBuffer buffer = this.buffer;
if(length > buffer.remaining()){
flush();
if(length >= buffer.capacity()){
writeFully(ByteBuffer.wrap(bytes, offset, length));
return;
}
}
buffer.put(bytes, offset, length);
}
@Override
public void write(int i) throws IOException {
byte b = (byte) (i & 0xff);
write(new byte[]{b});
ByteBuffer buffer = this.buffer;
if(!buffer.hasRemaining()){
flush();
}
buffer.put((byte) i);
}
/**
* Writes the buffered bytes to the channel at its current position.
* Must be called before the channel is used directly.
*/
@Override
public void close(){

public void flush() throws IOException {
ByteBuffer buffer = this.buffer;
if(buffer.position() == 0){
return;
}
buffer.flip();
writeFully(buffer);
buffer.clear();
}
private void writeFully(ByteBuffer byteBuffer) throws IOException {
FileChannel fileChannel = this.fileChannel;
while (byteBuffer.hasRemaining()){
fileChannel.write(byteBuffer);
}
}
@Override
public void close() throws IOException {
flush();
}

private static final int BUFFER_SIZE = 64 * 1024;
}
11 changes: 11 additions & 0 deletions src/main/java/com/reandroid/archive/io/ZipFileOutput.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ public File getFile() {
return file;
}
public void write(FileChannel input, long length) throws IOException{
flushOutputStream();
FileChannel fileChannel = getFileChannel();
long pos = fileChannel.position();

Expand All @@ -52,16 +53,19 @@ public void write(FileChannel input, long length) throws IOException{

@Override
public long position() throws IOException {
flushOutputStream();
return getFileChannel().position();
}
@Override
public void position(long pos) throws IOException {
flushOutputStream();
getFileChannel().position(pos);
}
@Override
public void close() throws IOException {
FileChannel fileChannel = this.fileChannel;
if(fileChannel != null){
flushOutputStream();
fileChannel.close();
}
}
Expand All @@ -87,6 +91,7 @@ private FileChannel getFileChannel() throws IOException {

@Override
public void write(InputStream inputStream) throws IOException {
flushOutputStream();
FileChannel fileChannel = getFileChannel();
long pos = fileChannel.position();
int bufferLength = 1024 * 1000 * 10;
Expand All @@ -111,6 +116,12 @@ public FileChannelOutputStream getOutputStream() throws IOException {
return outputStream;
}

private void flushOutputStream() throws IOException {
FileChannelOutputStream outputStream = this.outputStream;
if(outputStream != null){
outputStream.flush();
}
}

private static void initFile(File file) throws IOException{
if(file.isDirectory()){
Expand Down
Loading