().stream();
}
/**
diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/LimitedGZIPInputStream.java b/api/src/main/java/com/microsoft/gctoolkit/io/LimitedGZIPInputStream.java
new file mode 100644
index 000000000..fb277646e
--- /dev/null
+++ b/api/src/main/java/com/microsoft/gctoolkit/io/LimitedGZIPInputStream.java
@@ -0,0 +1,353 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+package com.microsoft.gctoolkit.io;
+
+import java.io.EOFException;
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PushbackInputStream;
+import java.nio.file.Path;
+import java.util.zip.CRC32;
+import java.util.zip.DataFormatException;
+import java.util.zip.Inflater;
+import java.util.zip.ZipException;
+
+final class LimitedGZIPInputStream extends InputStream {
+
+ private static final int BUFFER_SIZE = 8192;
+ private static final int FLAG_HEADER_CRC = 2;
+ private static final int FLAG_EXTRA = 4;
+ private static final int FLAG_NAME = 8;
+ private static final int FLAG_COMMENT = 16;
+ private static final int FLAG_RESERVED = 224;
+
+ private final CompressedCountingInputStream compressedSource;
+ private final PushbackInputStream source;
+ private final Inflater inflater = new Inflater(true);
+ private final CRC32 crc = new CRC32();
+ private final CRC32 headerCrc = new CRC32();
+ private final byte[] compressedBuffer = new byte[BUFFER_SIZE];
+ private final byte[] singleByte = new byte[1];
+ private final LogFileReadBudget budget;
+ private final LogFileReadLimits limits;
+ private final Path path;
+
+ private boolean memberOpen;
+ private boolean endOfStream;
+ private boolean closed;
+ private int lastInputLength;
+ private long headerBytes;
+ private long memberExpandedBytes;
+ private int memberCount;
+
+ LimitedGZIPInputStream(
+ InputStream inputStream,
+ LogFileReadBudget budget,
+ LogFileReadLimits limits,
+ Path path) {
+ compressedSource = new CompressedCountingInputStream(
+ inputStream,
+ limits.getMaxCompressedBytes(),
+ path);
+ source = new PushbackInputStream(compressedSource, BUFFER_SIZE);
+ this.budget = budget;
+ this.limits = limits;
+ this.path = path;
+ }
+
+ @Override
+ public int read() throws IOException {
+ int count = read(singleByte, 0, 1);
+ return count == -1 ? -1 : Byte.toUnsignedInt(singleByte[0]);
+ }
+
+ @Override
+ public int read(byte[] bytes, int offset, int length) throws IOException {
+ ensureOpen();
+ if (length == 0) {
+ return 0;
+ }
+
+ while (!endOfStream) {
+ if (!memberOpen && !openMember()) {
+ endOfStream = true;
+ return -1;
+ }
+
+ try {
+ int permittedLength = budget.nextReadLength(length);
+ int count = inflater.inflate(bytes, offset, permittedLength);
+ if (count > 0) {
+ crc.update(bytes, offset, count);
+ budget.record(count, path, null);
+ memberExpandedBytes += count;
+ budget.recordCompressedExpansion(
+ count,
+ compressedSource.getBytesRead(),
+ limits,
+ path,
+ null);
+ checkCompressionRatio();
+ return count;
+ }
+ if (inflater.finished()) {
+ finishMember();
+ } else if (inflater.needsDictionary()) {
+ throw new ZipException("GZIP member requires a preset dictionary: " + path);
+ } else if (inflater.needsInput()) {
+ fillInflater();
+ } else {
+ throw new ZipException("Unable to make progress while inflating " + path);
+ }
+ } catch (DataFormatException exception) {
+ ZipException failure = new ZipException("Invalid GZIP data: " + path);
+ failure.initCause(exception);
+ throw failure;
+ }
+ }
+ return -1;
+ }
+
+ private boolean openMember() throws IOException {
+ int magic1 = source.read();
+ if (magic1 == -1) {
+ return false;
+ }
+ if (++memberCount > limits.getMaxArchiveEntries()) {
+ throw new LogFileReadLimitExceededException(
+ LogFileReadLimitExceededException.LimitType.COMPRESSED_MEMBERS,
+ path,
+ null,
+ Integer.toString(limits.getMaxArchiveEntries()),
+ Integer.toString(memberCount));
+ }
+ headerCrc.reset();
+ headerCrc.update(magic1);
+ headerBytes = 1L;
+ checkHeaderBytes();
+ int magic2 = readHeaderByte();
+ if (magic1 != 0x1f || magic2 != 0x8b) {
+ throw new ZipException("Invalid GZIP header: " + path);
+ }
+ if (readHeaderByte() != 8) {
+ throw new ZipException("Unsupported GZIP compression method: " + path);
+ }
+ int flags = readHeaderByte();
+ if ((flags & FLAG_RESERVED) != 0) {
+ throw new ZipException("Invalid GZIP flags: " + path);
+ }
+ skipHeaderBytes(6);
+ if ((flags & FLAG_EXTRA) != 0) {
+ int extraLength = readHeaderByte() | (readHeaderByte() << 8);
+ skipHeaderBytes(extraLength);
+ }
+ if ((flags & FLAG_NAME) != 0) {
+ skipZeroTerminatedHeaderField();
+ }
+ if ((flags & FLAG_COMMENT) != 0) {
+ skipZeroTerminatedHeaderField();
+ }
+ if ((flags & FLAG_HEADER_CRC) != 0) {
+ int expectedHeaderCrc = readStoredHeaderByte() | (readStoredHeaderByte() << 8);
+ if (expectedHeaderCrc != ((int) headerCrc.getValue() & 0xffff)) {
+ throw new ZipException("Corrupt GZIP header: " + path);
+ }
+ }
+
+ inflater.reset();
+ crc.reset();
+ memberExpandedBytes = 0L;
+ lastInputLength = 0;
+ memberOpen = true;
+ return true;
+ }
+
+ private void fillInflater() throws IOException {
+ int count = source.read(compressedBuffer);
+ if (count == -1) {
+ throw new EOFException("Unexpected end of GZIP member: " + path);
+ }
+ lastInputLength = count;
+ inflater.setInput(compressedBuffer, 0, count);
+ }
+
+ private void finishMember() throws IOException {
+ int remaining = inflater.getRemaining();
+ if (remaining > 0) {
+ source.unread(compressedBuffer, lastInputLength - remaining, remaining);
+ }
+
+ long expectedCrc = readLittleEndianUnsignedInt();
+ long expectedSize = readLittleEndianUnsignedInt();
+ if (expectedCrc != crc.getValue()) {
+ throw new ZipException("Corrupt GZIP CRC: " + path);
+ }
+ if (expectedSize != (memberExpandedBytes & 0xffffffffL)) {
+ throw new ZipException("Corrupt GZIP size: " + path);
+ }
+ memberOpen = false;
+ }
+
+ private void checkCompressionRatio() {
+ if (memberExpandedBytes <= limits.getCompressionRatioGraceBytes()) {
+ return;
+ }
+ long compressedBytes = headerBytes + inflater.getBytesRead();
+ double ratio = compressedBytes == 0L
+ ? Double.POSITIVE_INFINITY
+ : (double) memberExpandedBytes / (double) compressedBytes;
+ if (ratio > limits.getMaxCompressionRatio()) {
+ throw new LogFileReadLimitExceededException(
+ LogFileReadLimitExceededException.LimitType.COMPRESSION_RATIO,
+ path,
+ null,
+ Double.toString(limits.getMaxCompressionRatio()),
+ Double.toString(ratio));
+ }
+ }
+
+ private int readHeaderByte() throws IOException {
+ int value = source.read();
+ if (value == -1) {
+ throw new EOFException("Unexpected end of GZIP header: " + path);
+ }
+ headerBytes++;
+ checkHeaderBytes();
+ headerCrc.update(value);
+ return value;
+ }
+
+ private int readStoredHeaderByte() throws IOException {
+ int value = source.read();
+ if (value == -1) {
+ throw new EOFException("Unexpected end of GZIP header: " + path);
+ }
+ headerBytes++;
+ checkHeaderBytes();
+ return value;
+ }
+
+ private void checkHeaderBytes() {
+ if (headerBytes > limits.getMaxGzipHeaderBytes()) {
+ throw new LogFileReadLimitExceededException(
+ LogFileReadLimitExceededException.LimitType.HEADER_BYTES,
+ path,
+ null,
+ Integer.toString(limits.getMaxGzipHeaderBytes()),
+ Long.toString(headerBytes));
+ }
+ }
+
+ private void skipHeaderBytes(int count) throws IOException {
+ for (int index = 0; index < count; index++) {
+ readHeaderByte();
+ }
+ }
+
+ private void skipZeroTerminatedHeaderField() throws IOException {
+ while (readHeaderByte() != 0) {
+ // Continue to the field terminator.
+ }
+ }
+
+ private long readLittleEndianUnsignedInt() throws IOException {
+ long value = 0L;
+ for (int index = 0; index < 4; index++) {
+ int next = source.read();
+ if (next == -1) {
+ throw new EOFException("Unexpected end of GZIP trailer: " + path);
+ }
+ value |= (long) next << (8 * index);
+ }
+ return value;
+ }
+
+ private void ensureOpen() throws IOException {
+ if (closed) {
+ throw new IOException("Stream closed");
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (!closed) {
+ closed = true;
+ inflater.end();
+ source.close();
+ }
+ }
+
+ private static final class CompressedCountingInputStream extends FilterInputStream {
+
+ private final long maximumBytes;
+ private final Path path;
+ private long bytesRead;
+
+ private CompressedCountingInputStream(
+ InputStream inputStream,
+ long maximumBytes,
+ Path path) {
+ super(inputStream);
+ this.maximumBytes = maximumBytes;
+ this.path = path;
+ }
+
+ private long getBytesRead() {
+ return bytesRead;
+ }
+
+ @Override
+ public int read() throws IOException {
+ int value = super.read();
+ if (value != -1) {
+ record(1L);
+ }
+ return value;
+ }
+
+ @Override
+ public int read(byte[] bytes, int offset, int length) throws IOException {
+ int count = super.read(bytes, offset, nextReadLength(length));
+ if (count > 0) {
+ record(count);
+ }
+ return count;
+ }
+
+ @Override
+ public long skip(long count) throws IOException {
+ long skipped = super.skip(nextReadLength(count));
+ record(skipped);
+ return skipped;
+ }
+
+ private int nextReadLength(int requestedLength) {
+ return (int) nextReadLength((long) requestedLength);
+ }
+
+ private long nextReadLength(long requestedLength) {
+ long remaining = maximumBytes - bytesRead;
+ long detectableLength = remaining == Long.MAX_VALUE ? Long.MAX_VALUE : remaining + 1L;
+ return Math.min(requestedLength, detectableLength);
+ }
+
+ private void record(long count) {
+ if (count <= 0L) {
+ return;
+ }
+ if (count > maximumBytes - bytesRead) {
+ long observed = bytesRead == Long.MAX_VALUE
+ ? Long.MAX_VALUE
+ : bytesRead + count;
+ throw new LogFileReadLimitExceededException(
+ LogFileReadLimitExceededException.LimitType.COMPRESSED_BYTES,
+ path,
+ null,
+ Long.toString(maximumBytes),
+ Long.toString(observed));
+ }
+ bytesRead += count;
+ }
+ }
+}
diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/LimitedInputStream.java b/api/src/main/java/com/microsoft/gctoolkit/io/LimitedInputStream.java
new file mode 100644
index 000000000..0e6707bbf
--- /dev/null
+++ b/api/src/main/java/com/microsoft/gctoolkit/io/LimitedInputStream.java
@@ -0,0 +1,95 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+package com.microsoft.gctoolkit.io;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.util.function.LongSupplier;
+
+final class LimitedInputStream extends FilterInputStream {
+
+ private final LogFileReadBudget budget;
+ private final LogFileReadLimits limits;
+ private final Path path;
+ private final String archiveEntry;
+ private final LongSupplier compressedBytes;
+ private long expandedBytes;
+ private long recordedCompressedBytes;
+
+ LimitedInputStream(
+ InputStream inputStream,
+ LogFileReadBudget budget,
+ LogFileReadLimits limits,
+ Path path,
+ String archiveEntry,
+ LongSupplier compressedBytes) {
+ super(inputStream);
+ this.budget = budget;
+ this.limits = limits;
+ this.path = path;
+ this.archiveEntry = archiveEntry;
+ this.compressedBytes = compressedBytes;
+ if (compressedBytes != null) {
+ budget.registerCompressedBytes(compressedBytes.getAsLong());
+ }
+ }
+
+ @Override
+ public int read() throws IOException {
+ int value = super.read();
+ if (value != -1) {
+ record(1L);
+ }
+ return value;
+ }
+
+ @Override
+ public int read(byte[] bytes, int offset, int length) throws IOException {
+ int permittedLength = budget.nextReadLength(length);
+ int count = super.read(bytes, offset, permittedLength);
+ if (count > 0) {
+ record(count);
+ }
+ return count;
+ }
+
+ @Override
+ public long skip(long count) throws IOException {
+ long skipped = super.skip(budget.nextSkipLength(count));
+ if (skipped > 0) {
+ record(skipped);
+ }
+ return skipped;
+ }
+
+ private void record(long count) {
+ budget.record(count, path, archiveEntry);
+ expandedBytes += count;
+ if (compressedBytes == null) {
+ return;
+ }
+ long currentCompressedBytes = compressedBytes.getAsLong();
+ if (currentCompressedBytes > recordedCompressedBytes) {
+ budget.registerCompressedBytes(currentCompressedBytes - recordedCompressedBytes);
+ recordedCompressedBytes = currentCompressedBytes;
+ }
+ budget.recordCompressedExpansion(count, -1L, limits, path, archiveEntry);
+ if (expandedBytes <= limits.getCompressionRatioGraceBytes()) {
+ return;
+ }
+
+ double ratio = currentCompressedBytes == 0L
+ ? Double.POSITIVE_INFINITY
+ : (double) expandedBytes / (double) currentCompressedBytes;
+ if (ratio > limits.getMaxCompressionRatio()) {
+ throw new LogFileReadLimitExceededException(
+ LogFileReadLimitExceededException.LimitType.COMPRESSION_RATIO,
+ path,
+ archiveEntry,
+ Double.toString(limits.getMaxCompressionRatio()),
+ Double.toString(ratio));
+ }
+ }
+}
diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadBudget.java b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadBudget.java
new file mode 100644
index 000000000..080e97242
--- /dev/null
+++ b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadBudget.java
@@ -0,0 +1,96 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+package com.microsoft.gctoolkit.io;
+
+import java.nio.file.Path;
+import java.util.concurrent.atomic.AtomicLong;
+
+final class LogFileReadBudget {
+
+ private final long maximumBytes;
+ private final AtomicLong expandedBytes = new AtomicLong();
+ private final AtomicLong ratioExpandedBytes = new AtomicLong();
+ private final AtomicLong ratioCompressedBytes = new AtomicLong();
+
+ LogFileReadBudget(long maximumBytes) {
+ this.maximumBytes = maximumBytes;
+ }
+
+ int nextReadLength(int requestedLength) {
+ long remaining = maximumBytes - expandedBytes.get();
+ long detectableLength = remaining == Long.MAX_VALUE ? Long.MAX_VALUE : remaining + 1L;
+ return (int) Math.min(requestedLength, detectableLength);
+ }
+
+ long nextSkipLength(long requestedLength) {
+ long remaining = maximumBytes - expandedBytes.get();
+ long detectableLength = remaining == Long.MAX_VALUE ? Long.MAX_VALUE : remaining + 1L;
+ return Math.min(requestedLength, detectableLength);
+ }
+
+ void record(long bytes, Path path, String archiveEntry) {
+ if (bytes <= 0) {
+ return;
+ }
+ while (true) {
+ long current = expandedBytes.get();
+ if (bytes > maximumBytes - current) {
+ long observed = current == maximumBytes ? maximumBytes : current + bytes;
+ throw new LogFileReadLimitExceededException(
+ LogFileReadLimitExceededException.LimitType.EXPANDED_BYTES,
+ path,
+ archiveEntry,
+ Long.toString(maximumBytes),
+ Long.toString(observed));
+ }
+ if (expandedBytes.compareAndSet(current, current + bytes)) {
+ return;
+ }
+ }
+ }
+
+ void rejectDeclaredSize(long declaredBytes, Path path, String archiveEntry) {
+ long current = expandedBytes.get();
+ if (declaredBytes > maximumBytes - current) {
+ throw new LogFileReadLimitExceededException(
+ LogFileReadLimitExceededException.LimitType.EXPANDED_BYTES,
+ path,
+ archiveEntry,
+ Long.toString(maximumBytes),
+ Long.toString(current + declaredBytes));
+ }
+ }
+
+ void registerCompressedBytes(long compressedBytes) {
+ if (compressedBytes > 0L) {
+ ratioCompressedBytes.addAndGet(compressedBytes);
+ }
+ }
+
+ void recordCompressedExpansion(
+ long expanded,
+ long observedCompressedBytes,
+ LogFileReadLimits limits,
+ Path path,
+ String archiveEntry) {
+ if (observedCompressedBytes >= 0L) {
+ ratioCompressedBytes.accumulateAndGet(observedCompressedBytes, Math::max);
+ }
+ long totalExpanded = ratioExpandedBytes.addAndGet(expanded);
+ if (totalExpanded <= limits.getCompressionRatioGraceBytes()) {
+ return;
+ }
+ long totalCompressed = ratioCompressedBytes.get();
+ double ratio = totalCompressed == 0L
+ ? Double.POSITIVE_INFINITY
+ : (double) totalExpanded / (double) totalCompressed;
+ if (ratio > limits.getMaxCompressionRatio()) {
+ throw new LogFileReadLimitExceededException(
+ LogFileReadLimitExceededException.LimitType.COMPRESSION_RATIO,
+ path,
+ archiveEntry,
+ Double.toString(limits.getMaxCompressionRatio()),
+ Double.toString(ratio));
+ }
+ }
+}
diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimitExceededException.java b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimitExceededException.java
new file mode 100644
index 000000000..4e012b08d
--- /dev/null
+++ b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimitExceededException.java
@@ -0,0 +1,104 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+package com.microsoft.gctoolkit.io;
+
+import java.nio.file.Path;
+
+/**
+ * Indicates that a garbage collection log exceeded a configured resource limit while being read.
+ *
+ * The exception is unchecked because stream I/O can fail lazily during a terminal stream
+ * operation, after {@link DataSource#stream()} has returned.
+ */
+public final class LogFileReadLimitExceededException extends RuntimeException {
+
+ private static final long serialVersionUID = 1L;
+
+ /** The resource limit that was exceeded. */
+ public enum LimitType {
+ EXPANDED_BYTES,
+ COMPRESSED_BYTES,
+ LINE_CHARACTERS,
+ COMPRESSION_RATIO,
+ ARCHIVE_ENTRIES,
+ ARCHIVE_METADATA_BYTES,
+ COMPRESSED_MEMBERS,
+ HEADER_BYTES
+ }
+
+ private final LimitType limitType;
+ private final Path path;
+ private final String archiveEntry;
+ private final String configuredLimit;
+ private final String observedValue;
+
+ LogFileReadLimitExceededException(
+ LimitType limitType,
+ Path path,
+ String archiveEntry,
+ String configuredLimit,
+ String observedValue) {
+ super(message(limitType, path, archiveEntry, configuredLimit, observedValue));
+ this.limitType = limitType;
+ this.path = path;
+ this.archiveEntry = archiveEntry;
+ this.configuredLimit = configuredLimit;
+ this.observedValue = observedValue;
+ }
+
+ private static String message(
+ LimitType limitType,
+ Path path,
+ String archiveEntry,
+ String configuredLimit,
+ String observedValue) {
+ String source = archiveEntry == null ? path.toString() : path + "!" + archiveEntry;
+ return "Log file read limit exceeded for " + source + ": "
+ + limitType + " limit " + configuredLimit + ", observed " + observedValue;
+ }
+
+ /**
+ * Returns the type of resource limit that was exceeded.
+ *
+ * @return limit type
+ */
+ public LimitType getLimitType() {
+ return limitType;
+ }
+
+ /**
+ * Returns the path being read when the limit was exceeded.
+ *
+ * @return source path
+ */
+ public Path getPath() {
+ return path;
+ }
+
+ /**
+ * Returns the ZIP entry being read, if any.
+ *
+ * @return ZIP entry name, or {@code null} for a top-level source
+ */
+ public String getArchiveEntry() {
+ return archiveEntry;
+ }
+
+ /**
+ * Returns the configured limit rendered in the units identified by {@link #getLimitType()}.
+ *
+ * @return configured limit
+ */
+ public String getConfiguredLimit() {
+ return configuredLimit;
+ }
+
+ /**
+ * Returns the observed value rendered in the units identified by {@link #getLimitType()}.
+ *
+ * @return observed value
+ */
+ public String getObservedValue() {
+ return observedValue;
+ }
+}
diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimits.java b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimits.java
new file mode 100644
index 000000000..b83ecd235
--- /dev/null
+++ b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileReadLimits.java
@@ -0,0 +1,295 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+package com.microsoft.gctoolkit.io;
+
+import java.util.Objects;
+
+/**
+ * Resource limits applied while reading garbage collection logs.
+ *
+ *
Limits are enforced for every top-level stream operation. Applications that need to process
+ * larger trusted logs can supply higher finite values through the log-file constructors.
+ */
+public final class LogFileReadLimits {
+
+ /** Default maximum number of bytes read from plaintext or emitted by decompression. */
+ public static final long DEFAULT_MAX_EXPANDED_BYTES = 2L * 1024L * 1024L * 1024L;
+
+ /** Default maximum number of bytes read from one compressed input. */
+ public static final long DEFAULT_MAX_COMPRESSED_BYTES = 1024L * 1024L * 1024L;
+
+ /** Default maximum number of decoded characters in one line. */
+ public static final int DEFAULT_MAX_LINE_CHARACTERS = 2 * 1024 * 1024;
+
+ /** Default maximum ratio of expanded bytes to compressed bytes. */
+ public static final double DEFAULT_MAX_COMPRESSION_RATIO = 100.0d;
+
+ /** Default expanded-byte threshold before compression-ratio enforcement begins. */
+ public static final long DEFAULT_COMPRESSION_RATIO_GRACE_BYTES = 1024L * 1024L;
+
+ /** Default maximum number of file entries in a rotating ZIP log. */
+ public static final int DEFAULT_MAX_ARCHIVE_ENTRIES = 1024;
+
+ /** Default maximum number of bytes in a ZIP central directory. */
+ public static final long DEFAULT_MAX_ARCHIVE_METADATA_BYTES = 16L * 1024L * 1024L;
+
+ /** Default maximum number of bytes in one GZIP member header. */
+ public static final int DEFAULT_MAX_GZIP_HEADER_BYTES = 64 * 1024;
+
+ private static final LogFileReadLimits DEFAULTS = new LogFileReadLimits(
+ DEFAULT_MAX_COMPRESSED_BYTES,
+ DEFAULT_MAX_EXPANDED_BYTES,
+ DEFAULT_MAX_LINE_CHARACTERS,
+ DEFAULT_MAX_COMPRESSION_RATIO,
+ DEFAULT_COMPRESSION_RATIO_GRACE_BYTES,
+ DEFAULT_MAX_ARCHIVE_ENTRIES,
+ DEFAULT_MAX_ARCHIVE_METADATA_BYTES,
+ DEFAULT_MAX_GZIP_HEADER_BYTES);
+
+ private final long maxCompressedBytes;
+ private final long maxExpandedBytes;
+ private final int maxLineCharacters;
+ private final double maxCompressionRatio;
+ private final long compressionRatioGraceBytes;
+ private final int maxArchiveEntries;
+ private final long maxArchiveMetadataBytes;
+ private final int maxGzipHeaderBytes;
+
+ /**
+ * Creates a set of finite log-file read limits.
+ *
+ * @param maxExpandedBytes maximum plaintext or expanded bytes per top-level stream operation
+ * @param maxLineCharacters maximum decoded characters in one line
+ * @param maxCompressionRatio maximum expanded-to-compressed byte ratio
+ * @param compressionRatioGraceBytes expanded bytes allowed before ratio enforcement begins
+ * @param maxArchiveEntries maximum file entries in a rotating ZIP log
+ */
+ public LogFileReadLimits(
+ long maxExpandedBytes,
+ int maxLineCharacters,
+ double maxCompressionRatio,
+ long compressionRatioGraceBytes,
+ int maxArchiveEntries) {
+ this(
+ maxExpandedBytes,
+ maxExpandedBytes,
+ maxLineCharacters,
+ maxCompressionRatio,
+ compressionRatioGraceBytes,
+ maxArchiveEntries,
+ DEFAULT_MAX_ARCHIVE_METADATA_BYTES,
+ DEFAULT_MAX_GZIP_HEADER_BYTES);
+ }
+
+ /**
+ * Creates a set of finite log-file read limits, including compressed-input limits.
+ *
+ * @param maxCompressedBytes maximum bytes read from one compressed input
+ * @param maxExpandedBytes maximum plaintext or expanded bytes per top-level stream operation
+ * @param maxLineCharacters maximum decoded characters in one line
+ * @param maxCompressionRatio maximum expanded-to-compressed byte ratio
+ * @param compressionRatioGraceBytes expanded bytes allowed before ratio enforcement begins
+ * @param maxArchiveEntries maximum ZIP entries, GZIP members, or rotating log segments
+ * @param maxGzipHeaderBytes maximum bytes in one GZIP member header
+ */
+ public LogFileReadLimits(
+ long maxCompressedBytes,
+ long maxExpandedBytes,
+ int maxLineCharacters,
+ double maxCompressionRatio,
+ long compressionRatioGraceBytes,
+ int maxArchiveEntries,
+ int maxGzipHeaderBytes) {
+ this(
+ maxCompressedBytes,
+ maxExpandedBytes,
+ maxLineCharacters,
+ maxCompressionRatio,
+ compressionRatioGraceBytes,
+ maxArchiveEntries,
+ DEFAULT_MAX_ARCHIVE_METADATA_BYTES,
+ maxGzipHeaderBytes);
+ }
+
+ /**
+ * Creates a set of finite log-file read limits, including archive metadata limits.
+ *
+ * @param maxCompressedBytes maximum bytes read from one compressed input
+ * @param maxExpandedBytes maximum plaintext or expanded bytes per top-level stream operation
+ * @param maxLineCharacters maximum decoded characters in one line
+ * @param maxCompressionRatio maximum expanded-to-compressed byte ratio
+ * @param compressionRatioGraceBytes expanded bytes allowed before ratio enforcement begins
+ * @param maxArchiveEntries maximum ZIP entries, GZIP members, or rotating log segments
+ * @param maxArchiveMetadataBytes maximum bytes in a ZIP central directory
+ * @param maxGzipHeaderBytes maximum bytes in one GZIP member header
+ */
+ public LogFileReadLimits(
+ long maxCompressedBytes,
+ long maxExpandedBytes,
+ int maxLineCharacters,
+ double maxCompressionRatio,
+ long compressionRatioGraceBytes,
+ int maxArchiveEntries,
+ long maxArchiveMetadataBytes,
+ int maxGzipHeaderBytes) {
+ if (maxCompressedBytes <= 0) {
+ throw new IllegalArgumentException("maxCompressedBytes must be positive");
+ }
+ if (maxExpandedBytes <= 0) {
+ throw new IllegalArgumentException("maxExpandedBytes must be positive");
+ }
+ if (maxLineCharacters <= 0) {
+ throw new IllegalArgumentException("maxLineCharacters must be positive");
+ }
+ if (!Double.isFinite(maxCompressionRatio) || maxCompressionRatio <= 0.0d) {
+ throw new IllegalArgumentException("maxCompressionRatio must be positive and finite");
+ }
+ if (compressionRatioGraceBytes <= 0) {
+ throw new IllegalArgumentException("compressionRatioGraceBytes must be positive");
+ }
+ if (maxArchiveEntries <= 0) {
+ throw new IllegalArgumentException("maxArchiveEntries must be positive");
+ }
+ if (maxArchiveMetadataBytes <= 0) {
+ throw new IllegalArgumentException("maxArchiveMetadataBytes must be positive");
+ }
+ if (maxGzipHeaderBytes <= 0) {
+ throw new IllegalArgumentException("maxGzipHeaderBytes must be positive");
+ }
+ this.maxCompressedBytes = maxCompressedBytes;
+ this.maxExpandedBytes = maxExpandedBytes;
+ this.maxLineCharacters = maxLineCharacters;
+ this.maxCompressionRatio = maxCompressionRatio;
+ this.compressionRatioGraceBytes = compressionRatioGraceBytes;
+ this.maxArchiveEntries = maxArchiveEntries;
+ this.maxArchiveMetadataBytes = maxArchiveMetadataBytes;
+ this.maxGzipHeaderBytes = maxGzipHeaderBytes;
+ }
+
+ /**
+ * Returns the secure default limits.
+ *
+ * @return immutable default limits
+ */
+ public static LogFileReadLimits defaults() {
+ return DEFAULTS;
+ }
+
+ /**
+ * Returns the maximum number of bytes read from one compressed input.
+ *
+ * @return maximum compressed bytes
+ */
+ public long getMaxCompressedBytes() {
+ return maxCompressedBytes;
+ }
+
+ /**
+ * Returns the maximum number of plaintext or expanded bytes per top-level stream operation.
+ *
+ * @return maximum expanded bytes
+ */
+ public long getMaxExpandedBytes() {
+ return maxExpandedBytes;
+ }
+
+ /**
+ * Returns the maximum number of decoded characters in one line.
+ *
+ * @return maximum line characters
+ */
+ public int getMaxLineCharacters() {
+ return maxLineCharacters;
+ }
+
+ /**
+ * Returns the maximum expanded-to-compressed byte ratio.
+ *
+ * @return maximum compression ratio
+ */
+ public double getMaxCompressionRatio() {
+ return maxCompressionRatio;
+ }
+
+ /**
+ * Returns the expanded-byte threshold before compression-ratio enforcement begins.
+ *
+ * @return compression-ratio grace bytes
+ */
+ public long getCompressionRatioGraceBytes() {
+ return compressionRatioGraceBytes;
+ }
+
+ /**
+ * Returns the maximum number of entries accepted in a rotating ZIP log.
+ *
+ * @return maximum archive entries
+ */
+ public int getMaxArchiveEntries() {
+ return maxArchiveEntries;
+ }
+
+ /**
+ * Returns the maximum number of bytes accepted in a ZIP central directory.
+ *
+ * @return maximum ZIP central-directory bytes
+ */
+ public long getMaxArchiveMetadataBytes() {
+ return maxArchiveMetadataBytes;
+ }
+
+ /**
+ * Returns the maximum number of bytes accepted in one GZIP member header.
+ *
+ * @return maximum GZIP header bytes
+ */
+ public int getMaxGzipHeaderBytes() {
+ return maxGzipHeaderBytes;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (!(other instanceof LogFileReadLimits)) {
+ return false;
+ }
+ LogFileReadLimits that = (LogFileReadLimits) other;
+ return maxCompressedBytes == that.maxCompressedBytes
+ && maxExpandedBytes == that.maxExpandedBytes
+ && maxLineCharacters == that.maxLineCharacters
+ && Double.compare(maxCompressionRatio, that.maxCompressionRatio) == 0
+ && compressionRatioGraceBytes == that.compressionRatioGraceBytes
+ && maxArchiveEntries == that.maxArchiveEntries
+ && maxArchiveMetadataBytes == that.maxArchiveMetadataBytes
+ && maxGzipHeaderBytes == that.maxGzipHeaderBytes;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ maxCompressedBytes,
+ maxExpandedBytes,
+ maxLineCharacters,
+ maxCompressionRatio,
+ compressionRatioGraceBytes,
+ maxArchiveEntries,
+ maxArchiveMetadataBytes,
+ maxGzipHeaderBytes);
+ }
+
+ @Override
+ public String toString() {
+ return "LogFileReadLimits{"
+ + "maxCompressedBytes=" + maxCompressedBytes
+ + ", maxExpandedBytes=" + maxExpandedBytes
+ + ", maxLineCharacters=" + maxLineCharacters
+ + ", maxCompressionRatio=" + maxCompressionRatio
+ + ", compressionRatioGraceBytes=" + compressionRatioGraceBytes
+ + ", maxArchiveEntries=" + maxArchiveEntries
+ + ", maxArchiveMetadataBytes=" + maxArchiveMetadataBytes
+ + ", maxGzipHeaderBytes=" + maxGzipHeaderBytes
+ + '}';
+ }
+}
diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/LogFileStreams.java b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileStreams.java
new file mode 100644
index 000000000..81c5c6f30
--- /dev/null
+++ b/api/src/main/java/com/microsoft/gctoolkit/io/LogFileStreams.java
@@ -0,0 +1,437 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+package com.microsoft.gctoolkit.io;
+
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipFile;
+import org.apache.commons.compress.utils.InputStreamStatistics;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UncheckedIOException;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.LongSupplier;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+import java.util.zip.CRC32;
+import java.util.zip.ZipException;
+
+final class LogFileStreams {
+
+ private LogFileStreams() {
+ }
+
+ static Stream plainText(
+ Path path,
+ LogFileReadLimits limits,
+ LogFileReadBudget budget) throws IOException {
+ return lines(
+ Files.newInputStream(path),
+ StandardCharsets.UTF_8,
+ limits,
+ budget,
+ path,
+ null,
+ null);
+ }
+
+ static Stream gzip(
+ Path path,
+ LogFileReadLimits limits,
+ LogFileReadBudget budget) throws IOException {
+ LimitedGZIPInputStream gzip = new LimitedGZIPInputStream(
+ Files.newInputStream(path),
+ budget,
+ limits,
+ path);
+ try {
+ return decodedLines(
+ gzip,
+ Charset.defaultCharset(),
+ limits,
+ path,
+ null);
+ } catch (RuntimeException | Error failure) {
+ closeAfterFailure(gzip, failure);
+ throw failure;
+ }
+ }
+
+ static Stream firstZipEntry(
+ Path path,
+ LogFileReadLimits limits,
+ LogFileReadBudget budget) throws IOException {
+ ZipEntryReference entry = zipEntries(path, limits).stream()
+ .filter(candidate -> !candidate.isDirectory())
+ .findFirst()
+ .orElse(null);
+ if (entry == null) {
+ throw new IOException("ZIP file contains no readable entries: " + path);
+ }
+ return zipEntry(path, entry, limits, budget);
+ }
+
+ static Stream zipEntry(
+ Path path,
+ String entryName,
+ LogFileReadLimits limits,
+ LogFileReadBudget budget) throws IOException {
+ ZipEntryReference entry = zipEntries(path, limits).stream()
+ .filter(candidate -> !candidate.isDirectory())
+ .filter(candidate -> candidate.getName().equals(entryName))
+ .findFirst()
+ .orElse(null);
+ if (entry == null) {
+ throw new IOException("ZIP entry not found: " + path + "!" + entryName);
+ }
+ return zipEntry(path, entry, limits, budget);
+ }
+
+ static Stream segment(
+ LogFileSegment segment,
+ LogFileReadLimits limits,
+ LogFileReadBudget budget) {
+ try {
+ if (segment instanceof GCLogFileZipSegment) {
+ return ((GCLogFileZipSegment) segment).stream(budget);
+ }
+ return plainText(segment.getPath(), limits, budget);
+ } catch (IOException exception) {
+ throw new UncheckedIOException(exception);
+ }
+ }
+
+ private static Stream lines(
+ InputStream inputStream,
+ Charset charset,
+ LogFileReadLimits limits,
+ LogFileReadBudget budget,
+ Path path,
+ String archiveEntry,
+ LongSupplier compressedBytes) {
+ LimitedInputStream limited = new LimitedInputStream(
+ inputStream,
+ budget,
+ limits,
+ path,
+ archiveEntry,
+ compressedBytes);
+ return decodedLines(limited, charset, limits, path, archiveEntry);
+ }
+
+ private static Stream decodedLines(
+ InputStream inputStream,
+ Charset charset,
+ LogFileReadLimits limits,
+ Path path,
+ String archiveEntry) {
+ BoundedLineSpliterator lines = new BoundedLineSpliterator(
+ new InputStreamReader(inputStream, charset),
+ limits.getMaxLineCharacters(),
+ path,
+ archiveEntry);
+ return StreamSupport.stream(lines, false).onClose(lines::closeUnchecked);
+ }
+
+ static List zipEntries(
+ Path path,
+ LogFileReadLimits limits) throws IOException {
+ SeekableByteChannel channel = openValidatedZipChannel(path, limits);
+ ZipFile zipFile;
+ try {
+ zipFile = openZipFile(channel);
+ } catch (IOException exception) {
+ closeAfterFailure(channel, exception);
+ throw exception;
+ }
+ try (ZipFile closeableZipFile = zipFile) {
+ List references = new ArrayList<>();
+ Map occurrences = new HashMap<>();
+ Enumeration entries = closeableZipFile.getEntries();
+ while (entries.hasMoreElements()) {
+ ZipArchiveEntry entry = entries.nextElement();
+ if (references.size() == limits.getMaxArchiveEntries()) {
+ throw new LogFileReadLimitExceededException(
+ LogFileReadLimitExceededException.LimitType.ARCHIVE_ENTRIES,
+ path,
+ null,
+ Integer.toString(limits.getMaxArchiveEntries()),
+ Integer.toString(references.size() + 1));
+ }
+ if (!entry.isDirectory()) {
+ validateEntryMetadata(closeableZipFile, entry, path);
+ }
+ int occurrence = occurrences.getOrDefault(entry.getName(), 0);
+ occurrences.put(entry.getName(), occurrence + 1);
+ references.add(new ZipEntryReference(
+ entry.getName(),
+ occurrence,
+ entry.isDirectory(),
+ entry.getMethod(),
+ entry.getCrc(),
+ entry.getCompressedSize(),
+ entry.getSize()));
+ }
+ return references;
+ }
+ }
+
+ static Stream zipEntry(
+ Path path,
+ ZipEntryReference reference,
+ LogFileReadLimits limits,
+ LogFileReadBudget budget) throws IOException {
+ SeekableByteChannel channel = openValidatedZipChannel(path, limits);
+ ZipFile zipFile;
+ try {
+ zipFile = openZipFile(channel);
+ } catch (IOException | RuntimeException | Error failure) {
+ closeAfterFailure(channel, failure);
+ throw failure;
+ }
+ try {
+ ZipArchiveEntry entry = resolveEntry(zipFile, reference, path);
+ validateEntryMetadata(zipFile, entry, path);
+ budget.rejectDeclaredSize(entry.getSize(), path, entry.getName());
+ if (entry.getSize() > limits.getCompressionRatioGraceBytes()) {
+ double ratio = entry.getCompressedSize() == 0L
+ ? Double.POSITIVE_INFINITY
+ : (double) entry.getSize() / (double) entry.getCompressedSize();
+ if (ratio > limits.getMaxCompressionRatio()) {
+ throw new LogFileReadLimitExceededException(
+ LogFileReadLimitExceededException.LimitType.COMPRESSION_RATIO,
+ path,
+ entry.getName(),
+ Double.toString(limits.getMaxCompressionRatio()),
+ Double.toString(ratio));
+ }
+ }
+
+ InputStream inputStream = zipFile.getInputStream(entry);
+ if (!(inputStream instanceof InputStreamStatistics)) {
+ throw new IOException("ZIP entry stream does not expose read statistics: "
+ + path + "!" + entry.getName());
+ }
+ ValidatedZipInputStream validated = new ValidatedZipInputStream(
+ inputStream,
+ (InputStreamStatistics) inputStream,
+ zipFile,
+ entry,
+ path);
+ return lines(
+ validated,
+ Charset.defaultCharset(),
+ limits,
+ budget,
+ path,
+ entry.getName(),
+ validated::getCompressedBytesRead);
+ } catch (IOException | RuntimeException | Error failure) {
+ closeAfterFailure(zipFile, failure);
+ throw failure;
+ }
+ }
+
+ private static SeekableByteChannel openValidatedZipChannel(
+ Path path,
+ LogFileReadLimits limits) throws IOException {
+ SeekableByteChannel channel = Files.newByteChannel(path, StandardOpenOption.READ);
+ try {
+ ZipArchivePreflight.validate(channel, path, limits);
+ return channel;
+ } catch (IOException | RuntimeException | Error failure) {
+ closeAfterFailure(channel, failure);
+ throw failure;
+ }
+ }
+
+ private static ZipFile openZipFile(SeekableByteChannel channel) throws IOException {
+ return ZipFile.builder().setSeekableByteChannel(channel).get();
+ }
+
+ private static ZipArchiveEntry resolveEntry(
+ ZipFile zipFile,
+ ZipEntryReference reference,
+ Path path) throws IOException {
+ int occurrence = 0;
+ for (ZipArchiveEntry entry : zipFile.getEntries(reference.getName())) {
+ if (occurrence++ == reference.occurrence) {
+ if (!reference.matches(entry)) {
+ throw new ZipException(
+ "ZIP entry metadata changed while reading: "
+ + path + "!" + reference.getName());
+ }
+ return entry;
+ }
+ }
+ throw new IOException("ZIP entry not found: " + path + "!" + reference.getName());
+ }
+
+ private static void validateEntryMetadata(
+ ZipFile zipFile,
+ ZipArchiveEntry entry,
+ Path path) throws ZipException {
+ if (!zipFile.canReadEntryData(entry)) {
+ throw new ZipException(
+ "Unsupported or encrypted ZIP entry: " + path + "!" + entry.getName());
+ }
+ if (entry.getSize() < 0L || entry.getCompressedSize() < 0L || entry.getCrc() < 0L) {
+ throw new ZipException(
+ "ZIP entry has incomplete metadata: " + path + "!" + entry.getName());
+ }
+ }
+
+ private static void closeAfterFailure(AutoCloseable closeable, Throwable failure) {
+ try {
+ closeable.close();
+ } catch (Exception closeException) {
+ failure.addSuppressed(closeException);
+ }
+ }
+
+ static final class ZipEntryReference {
+ private final String name;
+ private final int occurrence;
+ private final boolean directory;
+ private final int method;
+ private final long crc;
+ private final long compressedSize;
+ private final long expandedSize;
+
+ private ZipEntryReference(
+ String name,
+ int occurrence,
+ boolean directory,
+ int method,
+ long crc,
+ long compressedSize,
+ long expandedSize) {
+ this.name = name;
+ this.occurrence = occurrence;
+ this.directory = directory;
+ this.method = method;
+ this.crc = crc;
+ this.compressedSize = compressedSize;
+ this.expandedSize = expandedSize;
+ }
+
+ String getName() {
+ return name;
+ }
+
+ boolean isDirectory() {
+ return directory;
+ }
+
+ private boolean matches(ZipArchiveEntry entry) {
+ return directory == entry.isDirectory()
+ && method == entry.getMethod()
+ && crc == entry.getCrc()
+ && compressedSize == entry.getCompressedSize()
+ && expandedSize == entry.getSize();
+ }
+ }
+
+ private static final class ValidatedZipInputStream extends InputStream {
+ private final InputStream inputStream;
+ private final InputStreamStatistics statistics;
+ private final ZipFile zipFile;
+ private final ZipArchiveEntry entry;
+ private final Path path;
+ private final CRC32 crc = new CRC32();
+ private final byte[] singleByte = new byte[1];
+ private long expandedBytes;
+ private boolean validated;
+ private boolean closed;
+
+ private ValidatedZipInputStream(
+ InputStream inputStream,
+ InputStreamStatistics statistics,
+ ZipFile zipFile,
+ ZipArchiveEntry entry,
+ Path path) {
+ this.inputStream = inputStream;
+ this.statistics = statistics;
+ this.zipFile = zipFile;
+ this.entry = entry;
+ this.path = path;
+ }
+
+ long getCompressedBytesRead() {
+ return statistics.getCompressedCount();
+ }
+
+ @Override
+ public int read() throws IOException {
+ int count = read(singleByte, 0, 1);
+ return count == -1 ? -1 : Byte.toUnsignedInt(singleByte[0]);
+ }
+
+ @Override
+ public int read(byte[] bytes, int offset, int length) throws IOException {
+ int count = inputStream.read(bytes, offset, length);
+ if (count > 0) {
+ crc.update(bytes, offset, count);
+ expandedBytes += count;
+ } else if (count == -1) {
+ validate();
+ }
+ return count;
+ }
+
+ private void validate() throws ZipException {
+ if (validated) {
+ return;
+ }
+ validated = true;
+ if (expandedBytes != entry.getSize()) {
+ throw new ZipException(
+ "ZIP expanded size does not match central directory: " + path);
+ }
+ if (statistics.getCompressedCount() != entry.getCompressedSize()) {
+ throw new ZipException(
+ "ZIP compressed size does not match central directory: " + path);
+ }
+ if (crc.getValue() != entry.getCrc()) {
+ throw new ZipException("ZIP CRC does not match central directory: " + path);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ IOException failure = null;
+ try {
+ inputStream.close();
+ } catch (IOException exception) {
+ failure = exception;
+ }
+ try {
+ zipFile.close();
+ } catch (IOException exception) {
+ if (failure == null) {
+ failure = exception;
+ } else {
+ failure.addSuppressed(exception);
+ }
+ }
+ if (failure != null) {
+ throw failure;
+ }
+ }
+ }
+}
diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/RotatingGCLogFile.java b/api/src/main/java/com/microsoft/gctoolkit/io/RotatingGCLogFile.java
index 6155cd1f8..8568b333f 100644
--- a/api/src/main/java/com/microsoft/gctoolkit/io/RotatingGCLogFile.java
+++ b/api/src/main/java/com/microsoft/gctoolkit/io/RotatingGCLogFile.java
@@ -2,23 +2,12 @@
// Licensed under the MIT License.
package com.microsoft.gctoolkit.io;
-import java.io.BufferedReader;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.SequenceInputStream;
-import java.io.UncheckedIOException;
import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
-import java.util.Vector;
-import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
/**
* A collection of rotating GC log files. The collection will contain only those files that can be
@@ -26,89 +15,49 @@
*/
public class RotatingGCLogFile extends GCLogFile {
- private static final Logger LOGGER = Logger.getLogger(RotatingGCLogFile.class.getName());
-
/**
* Use the given path to find rotating log files. If the path is a file, the file name is used to match
* other files in the directory. If the path is a directory, all files in the directory are considered.
* @param path the path to a rotating log file, or to a directory containing rotating log files.
*/
public RotatingGCLogFile(Path path) {
- super(path);
+ this(path, LogFileReadLimits.defaults());
+ }
+
+ /**
+ * Use the given path to find rotating log files with explicit resource limits.
+ *
+ * @param path the path to a rotating log file, or to a directory containing rotating log files
+ * @param readLimits resource limits applied while inspecting and streaming the logs
+ */
+ public RotatingGCLogFile(Path path, LogFileReadLimits readLimits) {
+ super(path, readLimits);
}
private RotatingLogFileMetadata metaData;
public LogFileMetadata getMetaData() throws IOException {
if ( metaData == null)
- metaData = new RotatingLogFileMetadata(getPath());
+ metaData = new RotatingLogFileMetadata(getPath(), getReadLimits());
return metaData;
}
@Override
public Stream stream() throws IOException {
- if ( getMetaData().isDirectory() || getMetaData().isPlainText() || getMetaData().isZip())
+ LogFileReadBudget budget = new LogFileReadBudget(getReadLimits().getMaxExpandedBytes());
+ LogFileMetadata metadata = getMetaData();
+ if (metadata.isDirectory() || metadata.isPlainText() || metadata.isZip()) {
+ Stream segments = metaData.logFiles(budget);
return Stream.concat(
- getMetaData().logFiles()
- .flatMap(LogFileSegment::stream)
- .filter(Objects::nonNull)
- .map(String::trim)
- .filter(s -> s.length() > 0),
+ segments
+ .flatMap(segment -> LogFileStreams.segment(segment, getReadLimits(), budget))
+ .filter(Objects::nonNull)
+ .map(String::trim)
+ .filter(s -> s.length() > 0),
Stream.of(endOfData()));
- else // yes, this is returning an empty stream.
+ } else { // yes, this is returning an empty stream.
return Stream.of(endOfData());
- }
-
- private Stream stream(LogFileMetadata metadata, LinkedList segments) throws IOException {
- //todo: find rolling files....
- if (metadata.isPlainText() || metadata.isDirectory()) {
- switch (segments.size()) {
- case 0:
- String[] empty = new String[0];
- return Arrays.stream(empty);
- case 1:
- return segments.getFirst().stream();
- default:
- // This code removes elements from the list of segments, so work on a copy.
- LinkedList copySegments = new LinkedList<>(segments);
- Stream allSegments = Stream.concat(copySegments.removeFirst().stream(), copySegments.removeFirst().stream());
- while (!copySegments.isEmpty())
- allSegments = Stream.concat(allSegments, copySegments.removeFirst().stream());
- return allSegments;
- }
- } else if (metadata.isZip()) {
- return streamZipFile();
- } else if (metadata.isGZip()) {
- throw new IOException("Unable to stream GZip files. Please unzip and retry");
- }
- throw new IOException("Unrecognised file type");
- }
-
- @SuppressWarnings("resource")
- private Stream streamZipFile() throws IOException {
- ZipFile zipFile = new ZipFile(path.toFile());
- List entries = zipFile.stream().filter(entry -> !entry.isDirectory()).collect(Collectors.toList());
- Vector streams = new Vector<>();
-
- try {
- entries
- .stream()
- .map(entry -> {
- try {
- return zipFile.getInputStream(entry);
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- })
- .filter(Objects::nonNull)
- .forEach(streams::add);
- } catch (UncheckedIOException uioe) {
- throw uioe.getCause();
}
-
- SequenceInputStream sequenceInputStream = new SequenceInputStream(streams.elements());
-
- return new BufferedReader(new InputStreamReader(sequenceInputStream)).lines();
}
/**
diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/RotatingLogFileMetadata.java b/api/src/main/java/com/microsoft/gctoolkit/io/RotatingLogFileMetadata.java
index 2b2764be5..7f3ed328d 100644
--- a/api/src/main/java/com/microsoft/gctoolkit/io/RotatingLogFileMetadata.java
+++ b/api/src/main/java/com/microsoft/gctoolkit/io/RotatingLogFileMetadata.java
@@ -3,17 +3,16 @@
package com.microsoft.gctoolkit.io;
import java.io.IOException;
+import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedList;
import java.util.List;
-import java.util.logging.Level;
+import java.util.Objects;
import java.util.logging.Logger;
import java.util.stream.Stream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
import static java.util.stream.Collectors.toList;
@@ -25,6 +24,7 @@ public class RotatingLogFileMetadata extends LogFileMetadata {
private static final Logger LOG = Logger.getLogger(RotatingLogFileMetadata.class.getName());
private List segments;
+ private final LogFileReadLimits readLimits;
/**
* Creates metadata for a rotating garbage collection log source.
@@ -33,7 +33,19 @@ public class RotatingLogFileMetadata extends LogFileMetadata {
* @throws IOException if the path cannot be inspected
*/
public RotatingLogFileMetadata(Path path) throws IOException {
+ this(path, LogFileReadLimits.defaults());
+ }
+
+ /**
+ * Creates metadata for a rotating garbage collection log source with explicit resource limits.
+ *
+ * @param path path to a rotating log file, archive, or directory
+ * @param readLimits resource limits applied while inspecting log segments
+ * @throws IOException if the path cannot be inspected
+ */
+ public RotatingLogFileMetadata(Path path, LogFileReadLimits readLimits) throws IOException {
super(path);
+ this.readLimits = Objects.requireNonNull(readLimits, "readLimits");
}
/**
@@ -42,11 +54,15 @@ public RotatingLogFileMetadata(Path path) throws IOException {
* @return a stream of ordered log segments
*/
public Stream logFiles() {
+ return logFiles(new LogFileReadBudget(readLimits.getMaxExpandedBytes()));
+ }
+
+ Stream logFiles(LogFileReadBudget inspectionBudget) {
if ( segments == null) {
if ( isPlainText() || isDirectory())
- findSegments();
+ findSegments(inspectionBudget);
else if ( isZip())
- findZIPSegments();
+ findZIPSegments(inspectionBudget);
else {
LOG.warning("unknown log file format");
segments = new ArrayList<>();
@@ -55,17 +71,22 @@ else if ( isZip())
return segments.stream();
}
- private void findZIPSegments() {
- try (var zipfile = new ZipFile(getPath().toFile())) {
- segments = zipfile.stream()
- .filter(zipEntry -> !zipEntry.isDirectory())
- .map(ZipEntry::getName)
- .map(name -> new GCLogFileZipSegment(getPath(),name))
- .collect(toList());
+ private void findZIPSegments(LogFileReadBudget inspectionBudget) {
+ List entries;
+ try {
+ entries = LogFileStreams.zipEntries(getPath(), readLimits);
} catch (IOException ioe) {
- LOG.warning(ioe.getMessage());
+ throw new UncheckedIOException(ioe);
}
- orderSegments();
+ segments = entries.stream()
+ .filter(entry -> !entry.isDirectory())
+ .map(entry -> new GCLogFileZipSegment(
+ getPath(),
+ entry.getName(),
+ readLimits,
+ entry))
+ .collect(toList());
+ orderSegments(inspectionBudget);
}
/**
@@ -76,9 +97,9 @@ private void findZIPSegments() {
public int getNumberOfFiles() {
if ( this.segments == null)
if ( isZip())
- findZIPSegments();
+ findZIPSegments(new LogFileReadBudget(readLimits.getMaxExpandedBytes()));
else
- findSegments();
+ findSegments(new LogFileReadBudget(readLimits.getMaxExpandedBytes()));
return this.segments.size();
}
@@ -129,52 +150,77 @@ else if ( bits[bits.length - 1].matches("\\d+$"))
return base.toString();
}
- private void findSegments() {
- segments = new ArrayList<>();
- try {
+ private void findSegments(LogFileReadBudget inspectionBudget) {
+ try (Stream paths = Files.list(isDirectory() ? getPath() : getPath().getParent())) {
+ Stream matchingPaths = paths;
if (isDirectory()) {
- Files.list(getPath()).map(GCLogFileSegment::new).forEach(segments::add);
+ matchingPaths = paths;
+ } else {
+ matchingPaths = paths.filter(
+ file -> file.getFileName().toString().startsWith(getRootPattern()));
}
- else {
- Files.list(getPath().getParent())
- .filter(file -> file.getFileName().toString().startsWith(getRootPattern()))
- .map(p -> new GCLogFileSegment(p)).forEach(segments::add);
+ List segmentPaths = matchingPaths
+ .limit((long) readLimits.getMaxArchiveEntries() + 1L)
+ .collect(toList());
+ if (segmentPaths.size() > readLimits.getMaxArchiveEntries()) {
+ throw new LogFileReadLimitExceededException(
+ LogFileReadLimitExceededException.LimitType.ARCHIVE_ENTRIES,
+ getPath(),
+ null,
+ Integer.toString(readLimits.getMaxArchiveEntries()),
+ Integer.toString(segmentPaths.size()));
}
+ segments = segmentPaths.stream()
+ .map(path -> new GCLogFileSegment(path, readLimits))
+ .collect(toList());
} catch (IOException ioe) {
- LOG.log(Level.WARNING,"Unable to find log segments.", ioe);
+ throw new UncheckedIOException(ioe);
}
- orderSegments();
+ orderSegments(inspectionBudget);
}
- private void orderSegments() {
+ private void orderSegments(LogFileReadBudget inspectionBudget) {
if (segments.size() < 2) return;
- LinkedList orderedList = new LinkedList<>();
- List workingList = new ArrayList<>();
- workingList.addAll(segments);
-
- // Find current
String basePattern = getRootPattern();
- LogFileSegment current = workingList.stream()
+ LogFileSegment current = segments.stream()
.filter( segment -> segment.getSegmentName().endsWith(basePattern) || segment.getSegmentName().endsWith(".current"))
.findFirst().get();
+ LinkedList orderedList = new LinkedList<>();
orderedList.addLast(current);
- workingList = removeIneligibleSegments (workingList, current);
- while ( ! workingList.isEmpty()) {
- current = workingList.stream()
- .max(Comparator.comparing(LogFileSegment::getEndTime))
- .get();
- orderedList.addFirst(current);
- workingList = removeIneligibleSegments (workingList, current);
+ double nextStartTime = getStartTime(current, inspectionBudget);
+ List candidates = segments.stream()
+ .filter(segment -> segment != current)
+ .sorted(Comparator.comparing(
+ (LogFileSegment segment) -> getEndTime(segment, inspectionBudget))
+ .reversed())
+ .collect(toList());
+ for (LogFileSegment candidate : candidates) {
+ if (getEndTime(candidate, inspectionBudget) <= nextStartTime) {
+ orderedList.addFirst(candidate);
+ nextStartTime = getStartTime(candidate, inspectionBudget);
+ }
}
segments = orderedList;
}
- private List removeIneligibleSegments(final List logFileSegments, final LogFileSegment current) {
- return logFileSegments.stream()
- .filter( segment -> segment.getEndTime() <= current.getStartTime())
- .collect(toList());
+ private static double getStartTime(
+ LogFileSegment segment,
+ LogFileReadBudget inspectionBudget) {
+ if (segment instanceof GCLogFileZipSegment) {
+ return ((GCLogFileZipSegment) segment).getStartTime(inspectionBudget);
+ }
+ return ((GCLogFileSegment) segment).getStartTime(inspectionBudget);
+ }
+
+ private static double getEndTime(
+ LogFileSegment segment,
+ LogFileReadBudget inspectionBudget) {
+ if (segment instanceof GCLogFileZipSegment) {
+ return ((GCLogFileZipSegment) segment).getEndTime(inspectionBudget);
+ }
+ return ((GCLogFileSegment) segment).getEndTime(inspectionBudget);
}
}
diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/SingleGCLogFile.java b/api/src/main/java/com/microsoft/gctoolkit/io/SingleGCLogFile.java
index b3020b33e..dddaee0cb 100644
--- a/api/src/main/java/com/microsoft/gctoolkit/io/SingleGCLogFile.java
+++ b/api/src/main/java/com/microsoft/gctoolkit/io/SingleGCLogFile.java
@@ -2,18 +2,10 @@
// Licensed under the MIT License.
package com.microsoft.gctoolkit.io;
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Objects;
-import java.util.logging.Logger;
import java.util.stream.Stream;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
/**
* A single GC log file. If the file is a zip or gzip file,
@@ -21,23 +13,30 @@
*/
public class SingleGCLogFile extends GCLogFile {
- private static final Logger LOGGER = Logger.getLogger(SingleGCLogFile.class.getName());
+ private SingleLogFileMetadata metadata = null;
/**
* Constructor for a single, GC log file.
* @param path The path to the log file.
*/
-
- private SingleLogFileMetadata metadata = null;
-
public SingleGCLogFile(Path path) {
- super(path);
+ this(path, LogFileReadLimits.defaults());
+ }
+
+ /**
+ * Constructor for a single GC log file with explicit resource limits.
+ *
+ * @param path the path to the log file
+ * @param readLimits resource limits applied while streaming the log
+ */
+ public SingleGCLogFile(Path path, LogFileReadLimits readLimits) {
+ super(path, readLimits);
}
@Override
public LogFileMetadata getMetaData() throws IOException {
if (metadata == null) {
- metadata = new SingleLogFileMetadata(path);
+ metadata = new SingleLogFileMetadata(path, getReadLimits());
}
return metadata;
}
@@ -48,16 +47,17 @@ public Stream stream() throws IOException {
}
private Stream stream(LogFileMetadata metadata) throws IOException {
- Stream stream = null;
+ LogFileReadBudget budget = new LogFileReadBudget(getReadLimits().getMaxExpandedBytes());
+ Stream stream;
if (metadata.isPlainText()) {
- stream = Files.lines(metadata.getPath());
+ stream = LogFileStreams.plainText(metadata.getPath(), getReadLimits(), budget);
} else if (metadata.isZip()) {
- stream = streamZipFile(metadata.getPath());
+ stream = LogFileStreams.firstZipEntry(metadata.getPath(), getReadLimits(), budget);
} else if (metadata.isGZip()) {
- stream = streamGZipFile(metadata.getPath());
+ stream = LogFileStreams.gzip(metadata.getPath(), getReadLimits(), budget);
+ } else {
+ throw new IOException("Unable to read " + path);
}
- if ( stream == null)
- throw new IOException("Unable to read " + path.toString());
return Stream.concat(stream
.filter(Objects::nonNull)
.filter(line -> ! line.isBlank())
@@ -66,19 +66,4 @@ private Stream stream(LogFileMetadata metadata) throws IOException {
,Stream.of(endOfData()));
}
-
- private static Stream streamZipFile(Path path) throws IOException {
- ZipInputStream zipStream = new ZipInputStream(Files.newInputStream(path));
- ZipEntry entry;
- do {
- entry = zipStream.getNextEntry();
- } while (entry != null && entry.isDirectory());
- return new BufferedReader(new InputStreamReader(new BufferedInputStream(zipStream))).lines();
- }
-
- private static Stream streamGZipFile(Path path) throws IOException {
- GZIPInputStream gzipStream = new GZIPInputStream(Files.newInputStream(path));
- return new BufferedReader(new InputStreamReader(new BufferedInputStream(gzipStream))).lines();
- }
-
}
diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/SingleLogFileMetadata.java b/api/src/main/java/com/microsoft/gctoolkit/io/SingleLogFileMetadata.java
index 8be508c78..3737955f2 100644
--- a/api/src/main/java/com/microsoft/gctoolkit/io/SingleLogFileMetadata.java
+++ b/api/src/main/java/com/microsoft/gctoolkit/io/SingleLogFileMetadata.java
@@ -24,8 +24,12 @@ public class SingleLogFileMetadata extends LogFileMetadata {
* @throws IOException if the path cannot be inspected
*/
public SingleLogFileMetadata(Path path) throws IOException {
+ this(path, LogFileReadLimits.defaults());
+ }
+
+ SingleLogFileMetadata(Path path, LogFileReadLimits readLimits) throws IOException {
super(path);
- this.logFile = new GCLogFileSegment(path);
+ this.logFile = new GCLogFileSegment(path, readLimits);
}
/**
diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/ZipArchivePreflight.java b/api/src/main/java/com/microsoft/gctoolkit/io/ZipArchivePreflight.java
new file mode 100644
index 000000000..6d4d08fec
--- /dev/null
+++ b/api/src/main/java/com/microsoft/gctoolkit/io/ZipArchivePreflight.java
@@ -0,0 +1,284 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+package com.microsoft.gctoolkit.io;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.file.Path;
+import java.util.zip.ZipException;
+
+final class ZipArchivePreflight {
+
+ private static final long END_OF_CENTRAL_DIRECTORY = 0x06054b50L;
+ private static final long ZIP64_END_OF_CENTRAL_DIRECTORY = 0x06064b50L;
+ private static final long ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR = 0x07064b50L;
+ private static final long CENTRAL_DIRECTORY_ENTRY = 0x02014b50L;
+ private static final long CENTRAL_DIRECTORY_ARCHIVE_EXTRA_DATA = 0x08064b50L;
+ private static final long CENTRAL_DIRECTORY_DIGITAL_SIGNATURE = 0x05054b50L;
+ private static final int MAXIMUM_COMMENT_LENGTH = 0xffff;
+ private static final int END_OF_CENTRAL_DIRECTORY_SIZE = 22;
+
+ private ZipArchivePreflight() {
+ }
+
+ static void validate(
+ SeekableByteChannel channel,
+ Path path,
+ LogFileReadLimits limits) throws IOException {
+ Reader file = new Reader(channel);
+ if (file.length() > limits.getMaxCompressedBytes()) {
+ throw new LogFileReadLimitExceededException(
+ LogFileReadLimitExceededException.LimitType.COMPRESSED_BYTES,
+ path,
+ null,
+ Long.toString(limits.getMaxCompressedBytes()),
+ Long.toString(file.length()));
+ }
+
+ Directory directory = readDirectory(file, path);
+ if (directory.entryCount > limits.getMaxArchiveEntries()) {
+ throw limit(
+ LogFileReadLimitExceededException.LimitType.ARCHIVE_ENTRIES,
+ path,
+ limits.getMaxArchiveEntries(),
+ directory.entryCount);
+ }
+ if (directory.size > limits.getMaxArchiveMetadataBytes()) {
+ throw limit(
+ LogFileReadLimitExceededException.LimitType.ARCHIVE_METADATA_BYTES,
+ path,
+ limits.getMaxArchiveMetadataBytes(),
+ directory.size);
+ }
+
+ long directoryEnd = directory.offset + directory.size;
+ long entryCount = 0L;
+ long recordCount = 0L;
+ long maximumRecords = (long) limits.getMaxArchiveEntries() + 2L;
+ file.seek(directory.offset);
+ while (file.position() < directoryEnd) {
+ if (++recordCount > maximumRecords) {
+ throw limit(
+ LogFileReadLimitExceededException.LimitType.ARCHIVE_ENTRIES,
+ path,
+ maximumRecords,
+ recordCount);
+ }
+ long signature = file.readUnsignedInt();
+ if (signature == CENTRAL_DIRECTORY_ENTRY) {
+ entryCount++;
+ file.skip(24L, directoryEnd, path);
+ int nameLength = file.readUnsignedShort();
+ int extraLength = file.readUnsignedShort();
+ int commentLength = file.readUnsignedShort();
+ file.skip(
+ 12L + nameLength + extraLength + commentLength,
+ directoryEnd,
+ path);
+ } else if (signature == CENTRAL_DIRECTORY_ARCHIVE_EXTRA_DATA) {
+ file.skip(file.readUnsignedInt(), directoryEnd, path);
+ } else if (signature == CENTRAL_DIRECTORY_DIGITAL_SIGNATURE) {
+ file.skip(file.readUnsignedShort(), directoryEnd, path);
+ } else {
+ throw new ZipException("Invalid ZIP central directory record: " + path);
+ }
+ }
+ if (file.position() != directoryEnd || entryCount != directory.entryCount) {
+ throw new ZipException("ZIP central directory does not match its declared size: " + path);
+ }
+ }
+
+ private static Directory readDirectory(Reader file, Path path) throws IOException {
+ long endOffset = findEndOfCentralDirectory(file);
+ file.seek(endOffset + 4L);
+ int diskNumber = file.readUnsignedShort();
+ int centralDirectoryDisk = file.readUnsignedShort();
+ long entriesOnDisk = file.readUnsignedShort();
+ long totalEntries = file.readUnsignedShort();
+ long centralDirectorySize = file.readUnsignedInt();
+ long centralDirectoryOffset = file.readUnsignedInt();
+
+ if (diskNumber != 0 || centralDirectoryDisk != 0 || entriesOnDisk != totalEntries) {
+ throw new ZipException("Split ZIP archives are not supported: " + path);
+ }
+ if (totalEntries == 0xffffL
+ || centralDirectorySize == 0xffffffffL
+ || centralDirectoryOffset == 0xffffffffL) {
+ return readZip64Directory(file, path, endOffset);
+ }
+ return checkedDirectory(
+ path,
+ file.length(),
+ totalEntries,
+ centralDirectorySize,
+ centralDirectoryOffset);
+ }
+
+ private static long findEndOfCentralDirectory(Reader file) throws IOException {
+ long minimumOffset = Math.max(
+ 0L,
+ file.length() - END_OF_CENTRAL_DIRECTORY_SIZE - MAXIMUM_COMMENT_LENGTH);
+ for (long offset = file.length() - END_OF_CENTRAL_DIRECTORY_SIZE;
+ offset >= minimumOffset;
+ offset--) {
+ file.seek(offset);
+ if (file.readUnsignedInt() == END_OF_CENTRAL_DIRECTORY) {
+ file.seek(offset + 20L);
+ int commentLength = file.readUnsignedShort();
+ if (offset + END_OF_CENTRAL_DIRECTORY_SIZE + commentLength == file.length()) {
+ return offset;
+ }
+ }
+ }
+ throw new ZipException("ZIP end of central directory not found");
+ }
+
+ private static Directory readZip64Directory(
+ Reader file,
+ Path path,
+ long endOffset) throws IOException {
+ long locatorOffset = endOffset - 20L;
+ if (locatorOffset < 0L) {
+ throw new ZipException("ZIP64 locator not found: " + path);
+ }
+ file.seek(locatorOffset);
+ if (file.readUnsignedInt() != ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR
+ || file.readUnsignedInt() != 0L) {
+ throw new ZipException("Invalid or split ZIP64 locator: " + path);
+ }
+ long zip64Offset = file.readLong();
+ if (file.readUnsignedInt() != 1L
+ || zip64Offset < 0L
+ || zip64Offset > file.length() - 56L) {
+ throw new ZipException("Invalid ZIP64 locator: " + path);
+ }
+
+ file.seek(zip64Offset);
+ if (file.readUnsignedInt() != ZIP64_END_OF_CENTRAL_DIRECTORY) {
+ throw new ZipException("ZIP64 end of central directory not found: " + path);
+ }
+ long recordSize = file.readLong();
+ if (recordSize < 44L) {
+ throw new ZipException("Invalid ZIP64 end of central directory: " + path);
+ }
+ file.skip(4L, file.length(), path);
+ long diskNumber = file.readUnsignedInt();
+ long centralDirectoryDisk = file.readUnsignedInt();
+ long entriesOnDisk = file.readLong();
+ long totalEntries = file.readLong();
+ long centralDirectorySize = file.readLong();
+ long centralDirectoryOffset = file.readLong();
+ if (diskNumber != 0L
+ || centralDirectoryDisk != 0L
+ || entriesOnDisk != totalEntries) {
+ throw new ZipException("Split ZIP64 archives are not supported: " + path);
+ }
+ return checkedDirectory(
+ path,
+ file.length(),
+ totalEntries,
+ centralDirectorySize,
+ centralDirectoryOffset);
+ }
+
+ private static Directory checkedDirectory(
+ Path path,
+ long fileLength,
+ long entryCount,
+ long size,
+ long offset) throws ZipException {
+ if (entryCount < 0L || size < 0L || offset < 0L || offset > fileLength - size) {
+ throw new ZipException("Invalid ZIP central directory bounds: " + path);
+ }
+ return new Directory(entryCount, size, offset);
+ }
+
+ private static LogFileReadLimitExceededException limit(
+ LogFileReadLimitExceededException.LimitType limitType,
+ Path path,
+ long configured,
+ long observed) {
+ return new LogFileReadLimitExceededException(
+ limitType,
+ path,
+ null,
+ Long.toString(configured),
+ Long.toString(observed));
+ }
+
+ private static final class Reader {
+ private final SeekableByteChannel channel;
+ private final ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES)
+ .order(ByteOrder.LITTLE_ENDIAN);
+
+ private Reader(SeekableByteChannel channel) {
+ this.channel = channel;
+ }
+
+ private long length() throws IOException {
+ return channel.size();
+ }
+
+ private long position() throws IOException {
+ return channel.position();
+ }
+
+ private void seek(long position) throws IOException {
+ channel.position(position);
+ }
+
+ private int readUnsignedShort() throws IOException {
+ read(Short.BYTES);
+ return Short.toUnsignedInt(buffer.getShort());
+ }
+
+ private long readUnsignedInt() throws IOException {
+ read(Integer.BYTES);
+ return Integer.toUnsignedLong(buffer.getInt());
+ }
+
+ private long readLong() throws IOException {
+ read(Long.BYTES);
+ long value = buffer.getLong();
+ if (value < 0L) {
+ throw new ZipException("ZIP64 value is too large");
+ }
+ return value;
+ }
+
+ private void read(int length) throws IOException {
+ buffer.clear();
+ buffer.limit(length);
+ while (buffer.hasRemaining()) {
+ if (channel.read(buffer) == -1) {
+ throw new EOFException("Unexpected end of ZIP metadata");
+ }
+ }
+ buffer.flip();
+ }
+
+ private void skip(long count, long maximumOffset, Path path) throws IOException {
+ long current = position();
+ long target = current + count;
+ if (count < 0L || target < current || target > maximumOffset) {
+ throw new ZipException("Invalid ZIP central directory field length: " + path);
+ }
+ seek(target);
+ }
+ }
+
+ private static final class Directory {
+ private final long entryCount;
+ private final long size;
+ private final long offset;
+
+ private Directory(long entryCount, long size, long offset) {
+ this.entryCount = entryCount;
+ this.size = size;
+ this.offset = offset;
+ }
+ }
+}
diff --git a/api/src/main/java/com/microsoft/gctoolkit/jvm/AbstractJavaVirtualMachine.java b/api/src/main/java/com/microsoft/gctoolkit/jvm/AbstractJavaVirtualMachine.java
index bd7db30f2..33de2fd2a 100644
--- a/api/src/main/java/com/microsoft/gctoolkit/jvm/AbstractJavaVirtualMachine.java
+++ b/api/src/main/java/com/microsoft/gctoolkit/jvm/AbstractJavaVirtualMachine.java
@@ -15,6 +15,7 @@
import com.microsoft.gctoolkit.time.DateTimeStamp;
import java.io.IOException;
+import java.io.UncheckedIOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -23,6 +24,7 @@
import java.util.concurrent.Phaser;
import java.util.logging.Level;
import java.util.logging.Logger;
+import java.util.stream.Stream;
/**
* The base implementation of JavaVirtualMachine that uses the message API to feed
@@ -178,7 +180,9 @@ public void analyze(List> registeredAggregator
try {
if (finishLine.getRegisteredParties() > 0) {
- dataSource.stream().forEach(message -> dataSourceBus.publish(ChannelName.DATA_SOURCE, message));
+ try (Stream lines = dataSource.stream()) {
+ lines.forEach(message -> dataSourceBus.publish(ChannelName.DATA_SOURCE, message));
+ }
finishLine.awaitAdvance(0);
} else {
LOGGER.log(Level.INFO, "No Aggregations have been registered, DataSource will not be analysed.");
@@ -194,7 +198,7 @@ public void analyze(List> registeredAggregator
setEstimatedJVMStartTime(terminationRecord.estimatedStartTime());
});
} catch (IOException ioe) {
- LOGGER.log(Level.SEVERE, ioe.getMessage(), ioe);
+ throw new UncheckedIOException(ioe);
} finally {
dataSourceBus.close();
eventBus.close();
diff --git a/api/src/main/java/module-info.java b/api/src/main/java/module-info.java
index 8afa6d468..31437fd43 100644
--- a/api/src/main/java/module-info.java
+++ b/api/src/main/java/module-info.java
@@ -31,6 +31,7 @@
*/
module com.microsoft.gctoolkit.api {
requires java.logging;
+ requires org.apache.commons.compress;
exports com.microsoft.gctoolkit;
exports com.microsoft.gctoolkit.aggregator;
diff --git a/api/src/test/java/com/microsoft/gctoolkit/io/SingleGCLogFileReadLimitsTest.java b/api/src/test/java/com/microsoft/gctoolkit/io/SingleGCLogFileReadLimitsTest.java
new file mode 100644
index 000000000..4bc00af25
--- /dev/null
+++ b/api/src/test/java/com/microsoft/gctoolkit/io/SingleGCLogFileReadLimitsTest.java
@@ -0,0 +1,830 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+package com.microsoft.gctoolkit.io;
+
+import com.microsoft.gctoolkit.GCToolKit;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.zip.CRC32;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import static com.microsoft.gctoolkit.io.GCLogFile.END_OF_DATA_SENTINEL;
+import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.ARCHIVE_ENTRIES;
+import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.ARCHIVE_METADATA_BYTES;
+import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.COMPRESSED_BYTES;
+import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.COMPRESSED_MEMBERS;
+import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.COMPRESSION_RATIO;
+import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.EXPANDED_BYTES;
+import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.HEADER_BYTES;
+import static com.microsoft.gctoolkit.io.LogFileReadLimitExceededException.LimitType.LINE_CHARACTERS;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class SingleGCLogFileReadLimitsTest {
+
+ private static final int DEFAULT_MAX_LINE_CHARACTERS =
+ LogFileReadLimits.DEFAULT_MAX_LINE_CHARACTERS;
+ private static final int HIGHLY_COMPRESSIBLE_LINE_COUNT = 2048;
+ private static final String HIGHLY_COMPRESSIBLE_LINE = "a".repeat(1024);
+ private static final LogFileReadLimits SMALL_LIMITS =
+ new LogFileReadLimits(64 * 1024, 2048, 10_000.0d, 1024 * 1024, 16);
+
+ @TempDir
+ Path temporaryDirectory;
+
+ @Test
+ void rejectsLineLongerThanDefaultLimit() throws IOException {
+ Path log = temporaryDirectory.resolve("long-line.log");
+ Files.writeString(log, "a".repeat(DEFAULT_MAX_LINE_CHARACTERS + 1), StandardCharsets.UTF_8);
+
+ LogFileReadLimitExceededException failure = assertThrows(LogFileReadLimitExceededException.class, () -> {
+ try (Stream stream = new SingleGCLogFile(log).stream()) {
+ stream.findFirst();
+ }
+ });
+ assertEquals(LINE_CHARACTERS, failure.getLimitType());
+ assertEquals(log, failure.getPath());
+ }
+
+ @Test
+ void rejectsExcessiveGzipCompressionRatio() throws IOException {
+ Path log = temporaryDirectory.resolve("high-ratio.log.gz");
+ try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(
+ new GZIPOutputStream(Files.newOutputStream(log)), StandardCharsets.UTF_8))) {
+ for (int index = 0; index < HIGHLY_COMPRESSIBLE_LINE_COUNT; index++) {
+ writer.write(HIGHLY_COMPRESSIBLE_LINE);
+ writer.newLine();
+ }
+ }
+
+ LogFileReadLimitExceededException failure = assertThrows(LogFileReadLimitExceededException.class, () -> {
+ try (Stream stream = new SingleGCLogFile(log).stream()) {
+ stream.count();
+ }
+ });
+ assertEquals(COMPRESSION_RATIO, failure.getLimitType());
+ }
+
+ @Test
+ void acceptsLineAtConfiguredLimitAndCommonLineEndings() throws IOException {
+ Path log = temporaryDirectory.resolve("line-endings.log");
+ Files.writeString(log, "ab\r\ncd\ref\néé", StandardCharsets.UTF_8);
+ LogFileReadLimits limits = new LogFileReadLimits(128, 2, 100.0d, 64, 16);
+
+ try (Stream stream = new SingleGCLogFile(log, limits).stream()) {
+ assertEquals(
+ List.of("ab", "cd", "ef", "éé", END_OF_DATA_SENTINEL),
+ stream.collect(Collectors.toList()));
+ }
+ }
+
+ @Test
+ void rejectsPlaintextExpandedBytesOverConfiguredLimit() throws IOException {
+ Path log = temporaryDirectory.resolve("expanded.log");
+ Files.writeString(log, repeatedLines(80, 1023), StandardCharsets.UTF_8);
+
+ assertLimit(EXPANDED_BYTES, () -> {
+ try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) {
+ stream.count();
+ }
+ });
+ }
+
+ @Test
+ void rejectsGzipExpandedBytesOverConfiguredLimitAcrossMembers() throws IOException {
+ Path log = temporaryDirectory.resolve("concatenated.log.gz");
+ writeGzipMember(log, repeatedLines(40, 1023), false);
+ writeGzipMember(log, repeatedLines(40, 1023), true);
+
+ assertLimit(EXPANDED_BYTES, () -> {
+ try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) {
+ stream.count();
+ }
+ });
+ }
+
+ @Test
+ void acceptsExpandedBytesAtConfiguredLimit() throws IOException {
+ Path log = temporaryDirectory.resolve("expanded-boundary.log");
+ Files.writeString(log, repeatedLines(64, 1023), StandardCharsets.UTF_8);
+
+ try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) {
+ assertEquals(65L, stream.count());
+ }
+ }
+
+ @Test
+ void rejectsZipExpandedBytesOverConfiguredLimit() throws IOException {
+ Path log = temporaryDirectory.resolve("expanded.log.zip");
+ writeZip(log, Map.of("gc.log", repeatedLines(80, 1023)));
+
+ assertLimit(EXPANDED_BYTES, () -> {
+ try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) {
+ stream.count();
+ }
+ });
+ }
+
+ @Test
+ void rejectsLongLineInGzip() throws IOException {
+ Path log = temporaryDirectory.resolve("long-line.log.gz");
+ writeGzipMember(log, "a".repeat(2049), false);
+
+ assertLimit(LINE_CHARACTERS, () -> {
+ try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) {
+ stream.findFirst();
+ }
+ });
+ }
+
+ @Test
+ void rejectsLongLineInZip() throws IOException {
+ Path log = temporaryDirectory.resolve("long-line.log.zip");
+ writeZip(log, Map.of("gc.log", "a".repeat(2049)));
+
+ assertLimit(LINE_CHARACTERS, () -> {
+ try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) {
+ stream.findFirst();
+ }
+ });
+ }
+
+ @Test
+ void rejectsExcessiveZipCompressionRatio() throws IOException {
+ Path log = temporaryDirectory.resolve("high-ratio.log.zip");
+ writeZip(log, Map.of("gc.log", repeatedLines(2048, 1023)));
+
+ assertLimit(COMPRESSION_RATIO, () -> {
+ try (Stream stream = new SingleGCLogFile(log).stream()) {
+ stream.count();
+ }
+ });
+ }
+
+ @Test
+ void acceptsCompressionRatioWithinGraceThreshold() throws IOException {
+ Path log = temporaryDirectory.resolve("ratio-grace.log.gz");
+ writeGzipMember(log, "line\n", false);
+ LogFileReadLimits limits = new LogFileReadLimits(1024, 128, 0.1d, 128, 16);
+
+ try (Stream stream = new SingleGCLogFile(log, limits).stream()) {
+ assertEquals(2L, stream.count());
+ }
+ }
+
+ @Test
+ void rejectsHighRatioConcatenatedGzipMemberAfterLowRatioMember() throws IOException {
+ Path log = temporaryDirectory.resolve("mixed-ratio-members.log.gz");
+ writeGzipMember(log, randomAsciiLine(50_000), false);
+ writeGzipMember(log, "a".repeat(100_000) + "\n", true);
+ LogFileReadLimits limits = new LogFileReadLimits(256 * 1024, 200_000, 10.0d, 1024, 16);
+
+ assertLimit(COMPRESSION_RATIO, () -> {
+ try (Stream stream = new SingleGCLogFile(log, limits).stream()) {
+ stream.count();
+ }
+ });
+ }
+
+ @Test
+ void appliesCompressionRatioGraceOnceAcrossGzipMembers() throws IOException {
+ Path log = temporaryDirectory.resolve("split-grace-members.log.gz");
+ for (int index = 0; index < 10; index++) {
+ writeGzipMember(log, "a".repeat(1024), index != 0);
+ }
+ LogFileReadLimits limits = new LogFileReadLimits(32 * 1024, 16 * 1024, 2.0d, 1024, 16);
+
+ assertLimit(COMPRESSION_RATIO, () -> {
+ try (Stream stream = new SingleGCLogFile(log, limits).stream()) {
+ stream.count();
+ }
+ });
+ }
+
+ @Test
+ void appliesCompressionRatioGraceOnceAcrossZipEntries() throws IOException {
+ Path log = temporaryDirectory.resolve("split-grace-entries.zip");
+ writeZip(log, Map.of(
+ "gc.log.0", "[1.0s][info][gc] " + "a".repeat(800) + "\n",
+ "gc.log.1", "[2.0s][info][gc] " + "a".repeat(800) + "\n",
+ "gc.log", "[3.0s][info][gc] " + "a".repeat(800) + "\n"));
+ LogFileReadLimits limits = new LogFileReadLimits(32 * 1024, 2048, 2.0d, 1024, 16);
+
+ assertLimit(
+ COMPRESSION_RATIO,
+ () -> new RotatingGCLogFile(log, limits).getMetaData().getNumberOfFiles());
+ }
+
+ @Test
+ void validatesGzipHeaderCrc() throws IOException {
+ Path valid = temporaryDirectory.resolve("valid-header-crc.log.gz");
+ writeGzipWithHeaderCrc(valid, "line\n", false);
+ try (Stream stream = new SingleGCLogFile(valid, SMALL_LIMITS).stream()) {
+ assertEquals(2L, stream.count());
+ }
+
+ Path invalid = temporaryDirectory.resolve("invalid-header-crc.log.gz");
+ writeGzipWithHeaderCrc(invalid, "line\n", true);
+ assertThrows(UncheckedIOException.class, () -> {
+ try (Stream stream = new SingleGCLogFile(invalid, SMALL_LIMITS).stream()) {
+ stream.count();
+ }
+ });
+ }
+
+ @Test
+ void rejectsExcessiveGzipMembersEvenWhenEmpty() throws IOException {
+ Path log = temporaryDirectory.resolve("empty-members.log.gz");
+ for (int index = 0; index < 5; index++) {
+ writeGzipMember(log, "", index != 0);
+ }
+ LogFileReadLimits limits =
+ new LogFileReadLimits(64 * 1024, 64 * 1024, 1024, 100.0d, 1024, 4, 1024);
+
+ assertLimit(COMPRESSED_MEMBERS, () -> {
+ try (Stream stream = new SingleGCLogFile(log, limits).stream()) {
+ stream.count();
+ }
+ });
+ }
+
+ @Test
+ void rejectsOversizedGzipHeaderField() throws IOException {
+ Path log = temporaryDirectory.resolve("long-header.log.gz");
+ writeGzipWithFileName(log, "a".repeat(64), "line\n");
+ LogFileReadLimits limits =
+ new LogFileReadLimits(64 * 1024, 64 * 1024, 1024, 100.0d, 1024, 16, 32);
+
+ assertLimit(HEADER_BYTES, () -> {
+ try (Stream stream = new SingleGCLogFile(log, limits).stream()) {
+ stream.count();
+ }
+ });
+ }
+
+ @Test
+ void rejectsOversizedCompressedGzipInput() throws IOException {
+ Path log = temporaryDirectory.resolve("compressed-input.log.gz");
+ writeGzipMember(log, randomAsciiLine(4096), false);
+ LogFileReadLimits limits =
+ new LogFileReadLimits(512, 16 * 1024, 16 * 1024, 100.0d, 1024, 16, 1024);
+
+ assertLimit(COMPRESSED_BYTES, () -> {
+ try (Stream stream = new SingleGCLogFile(log, limits).stream()) {
+ stream.count();
+ }
+ });
+ }
+
+ @Test
+ void ignoresPayloadOfZipDirectoryEntry() throws IOException {
+ Path log = temporaryDirectory.resolve("directory-payload.zip");
+ try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(log))) {
+ zip.putNextEntry(new ZipEntry("ignored/"));
+ zip.write("a".repeat(128 * 1024).getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ zip.putNextEntry(new ZipEntry("gc.log"));
+ zip.write("line\n".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ }
+ LogFileReadLimits limits = new LogFileReadLimits(1024, 128, 2.0d, 128, 16);
+
+ try (Stream stream = new SingleGCLogFile(log, limits).stream()) {
+ assertEquals(2L, stream.count());
+ }
+ }
+
+ @Test
+ void readsFirstFileAfterZipDirectory() throws IOException {
+ Path log = temporaryDirectory.resolve("directory-first.zip");
+ try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(log))) {
+ zip.putNextEntry(new ZipEntry("logs/"));
+ zip.closeEntry();
+ zip.putNextEntry(new ZipEntry("logs/gc.log"));
+ zip.write("line\n".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ }
+
+ try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) {
+ assertEquals(List.of("line", END_OF_DATA_SENTINEL), stream.collect(Collectors.toList()));
+ }
+ }
+
+ @Test
+ void readsAndValidatesStoredZipEntry() throws IOException {
+ Path log = temporaryDirectory.resolve("stored.zip");
+ byte[] content = "line\n".getBytes(StandardCharsets.UTF_8);
+ CRC32 crc = new CRC32();
+ crc.update(content);
+ try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(log))) {
+ ZipEntry entry = new ZipEntry("gc.log");
+ entry.setMethod(ZipEntry.STORED);
+ entry.setSize(content.length);
+ entry.setCompressedSize(content.length);
+ entry.setCrc(crc.getValue());
+ zip.putNextEntry(entry);
+ zip.write(content);
+ zip.closeEntry();
+ }
+
+ try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) {
+ assertEquals(2L, stream.count());
+ }
+ }
+
+ @Test
+ void readsStoredZipEntryWithDataDescriptor() throws IOException {
+ Path log = temporaryDirectory.resolve("stored-descriptor.zip");
+ byte[] content = "line\n".getBytes(StandardCharsets.UTF_8);
+ CRC32 crc = new CRC32();
+ crc.update(content);
+ try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(log))) {
+ ZipEntry entry = new ZipEntry("gc.log");
+ entry.setMethod(ZipEntry.STORED);
+ entry.setSize(content.length);
+ entry.setCompressedSize(content.length);
+ entry.setCrc(crc.getValue());
+ zip.putNextEntry(entry);
+ zip.write(content);
+ zip.closeEntry();
+ }
+ Files.write(log, withStoredDataDescriptor(Files.readAllBytes(log)));
+
+ try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) {
+ assertEquals(2L, stream.count());
+ }
+ }
+
+ @Test
+ void rejectsEmptyZip() throws IOException {
+ Path log = temporaryDirectory.resolve("empty.zip");
+ try (ZipOutputStream ignored = new ZipOutputStream(Files.newOutputStream(log))) {
+ // Empty archive.
+ }
+
+ assertThrows(IOException.class, () -> new SingleGCLogFile(log, SMALL_LIMITS).stream());
+ }
+
+ @Test
+ void rejectsZipWithStaleCentralDirectoryCrc() throws IOException {
+ Path log = temporaryDirectory.resolve("stale-central-crc.zip");
+ writeZip(log, Map.of("gc.log", "line\n"));
+ byte[] archive = Files.readAllBytes(log);
+ int centralDirectory = findSignature(archive, 0x02014b50);
+ archive[centralDirectory + 16] ^= 1;
+ Files.write(log, archive);
+
+ assertThrows(UncheckedIOException.class, () -> {
+ try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) {
+ stream.count();
+ }
+ });
+ }
+
+ @Test
+ void rejectsZipWithForgedCentralDirectoryCompressedSize() throws IOException {
+ Path log = temporaryDirectory.resolve("forged-compressed-size.zip");
+ writeZip(log, Map.of("gc.log", "a".repeat(8192) + "\n"));
+ byte[] archive = Files.readAllBytes(log);
+ int centralDirectory = findSignature(archive, 0x02014b50);
+ putUnsignedInt(archive, centralDirectory + 20, 1024 * 1024);
+ Files.write(log, archive);
+ LogFileReadLimits limits = new LogFileReadLimits(16 * 1024, 16 * 1024, 10_000.0d, 1024, 16);
+
+ assertThrows(IOException.class, () -> new SingleGCLogFile(log, limits).stream());
+ }
+
+ @Test
+ void rejectsOversizedZip64DeclaredEntryCount() throws IOException {
+ Path log = temporaryDirectory.resolve("forged-zip64-count.zip");
+ writeZip(log, Map.of("gc.log", "line\n"));
+ Files.write(log, withZip64EntryCount(Files.readAllBytes(log), 500_000_000L));
+
+ LogFileReadLimitExceededException failure = assertThrows(
+ LogFileReadLimitExceededException.class,
+ () -> new SingleGCLogFile(log, SMALL_LIMITS).stream());
+ assertEquals(ARCHIVE_ENTRIES, failure.getLimitType());
+ }
+
+ @Test
+ void rejectsOversizedCompressedZipInput() throws IOException {
+ Path log = temporaryDirectory.resolve("compressed-input.zip");
+ writeZip(log, Map.of("gc.log", randomAsciiLine(4096)));
+ LogFileReadLimits limits =
+ new LogFileReadLimits(512, 16 * 1024, 16 * 1024, 100.0d, 1024, 16, 1024);
+
+ assertLimit(COMPRESSED_BYTES, () -> new SingleGCLogFile(log, limits).stream());
+ }
+
+ @Test
+ void rejectsExcessiveZipControlRecords() throws IOException {
+ Path log = temporaryDirectory.resolve("control-records.zip");
+ Files.write(log, zipWithArchiveExtraRecords(5));
+ LogFileReadLimits limits =
+ new LogFileReadLimits(64 * 1024, 64 * 1024, 1024, 100.0d, 1024, 2, 1024);
+
+ assertLimit(ARCHIVE_ENTRIES, () -> new SingleGCLogFile(log, limits).stream());
+ }
+
+ @Test
+ void rejectsOversizedZipCentralDirectoryBeforeIndexing() throws IOException {
+ Path log = temporaryDirectory.resolve("central-directory.zip");
+ writeZip(log, Map.of("gc.log", "line\n"));
+ LogFileReadLimits limits = new LogFileReadLimits(
+ 64 * 1024,
+ 64 * 1024,
+ 1024,
+ 100.0d,
+ 1024,
+ 16,
+ 32,
+ 1024);
+
+ assertLimit(ARCHIVE_METADATA_BYTES, () -> new SingleGCLogFile(log, limits).stream());
+ }
+
+ @Test
+ void rejectsExcessiveDirectorySegments() throws IOException {
+ Path directory = temporaryDirectory.resolve("rotating");
+ Files.createDirectory(directory);
+ Files.writeString(directory.resolve("gc.log"), "[3.0s][info][gc] current\n");
+ Files.writeString(directory.resolve("gc.log.0"), "[1.0s][info][gc] first\n");
+ Files.writeString(directory.resolve("gc.log.1"), "[2.0s][info][gc] second\n");
+ LogFileReadLimits limits =
+ new LogFileReadLimits(64 * 1024, 64 * 1024, 1024, 100.0d, 1024, 2, 1024);
+
+ assertLimit(
+ ARCHIVE_ENTRIES,
+ () -> new RotatingGCLogFile(directory, limits).getMetaData().getNumberOfFiles());
+ }
+
+ @Test
+ void doesNotEmitSentinelAfterLimitFailure() throws IOException {
+ Path log = temporaryDirectory.resolve("sentinel.log");
+ Files.writeString(log, "ok\n" + "a".repeat(2049), StandardCharsets.UTF_8);
+ List consumed = new ArrayList<>();
+
+ assertLimit(LINE_CHARACTERS, () -> {
+ try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) {
+ stream.forEach(consumed::add);
+ }
+ });
+ assertEquals(List.of("ok"), consumed);
+ assertFalse(consumed.contains(END_OF_DATA_SENTINEL));
+ }
+
+ @Test
+ void closesFileAfterLimitFailure() throws IOException {
+ Path log = temporaryDirectory.resolve("closed-after-failure.log");
+ Files.writeString(log, "a".repeat(2049), StandardCharsets.UTF_8);
+
+ assertLimit(LINE_CHARACTERS, () -> {
+ try (Stream stream = new SingleGCLogFile(log, SMALL_LIMITS).stream()) {
+ stream.findFirst();
+ }
+ });
+ assertDoesNotThrow(() -> Files.delete(log));
+ assertFalse(Files.exists(log));
+ }
+
+ @Test
+ void analysisPreservesLimitFailure() throws IOException {
+ Path log = temporaryDirectory.resolve("analysis-limit.log");
+ Files.writeString(log, "a".repeat(2049), StandardCharsets.UTF_8);
+
+ assertLimit(
+ LINE_CHARACTERS,
+ () -> new GCToolKit().analyze(new SingleGCLogFile(log, SMALL_LIMITS)));
+ }
+
+ @Test
+ void rejectsAggregateExpandedBytesAcrossRotatingZipEntries() throws IOException {
+ Path log = temporaryDirectory.resolve("gc.log.zip");
+ writeZip(log, Map.of(
+ "gc.log.0", timestampedLines(1.0d, 40, 1000),
+ "gc.log", timestampedLines(2.0d, 40, 1000)));
+
+ assertLimit(EXPANDED_BYTES, () -> {
+ try (Stream stream = new RotatingGCLogFile(log, SMALL_LIMITS).stream()) {
+ stream.count();
+ }
+ });
+ }
+
+ @Test
+ void rejectsExcessiveRotatingZipEntries() throws IOException {
+ Path log = temporaryDirectory.resolve("many-entries.zip");
+ writeZip(log, Map.of(
+ "gc.log.0", "[1.0s][info][gc] first\n",
+ "gc.log.1", "[2.0s][info][gc] second\n",
+ "gc.log", "[3.0s][info][gc] current\n"));
+ LogFileReadLimits limits = new LogFileReadLimits(64 * 1024, 2048, 100.0d, 1024, 2);
+
+ LogFileReadLimitExceededException failure = assertThrows(
+ LogFileReadLimitExceededException.class,
+ () -> new RotatingGCLogFile(log, limits).getMetaData().getNumberOfFiles());
+ assertEquals(ARCHIVE_ENTRIES, failure.getLimitType());
+ }
+
+ @Test
+ void validatesLimitConfiguration() {
+ assertThrows(IllegalArgumentException.class,
+ () -> new LogFileReadLimits(0, 1, 1.0d, 1, 1));
+ assertThrows(IllegalArgumentException.class,
+ () -> new LogFileReadLimits(1, 0, 1.0d, 1, 1));
+ assertThrows(IllegalArgumentException.class,
+ () -> new LogFileReadLimits(1, 1, Double.POSITIVE_INFINITY, 1, 1));
+ assertThrows(IllegalArgumentException.class,
+ () -> new LogFileReadLimits(1, 1, 1.0d, 0, 1));
+ assertThrows(IllegalArgumentException.class,
+ () -> new LogFileReadLimits(1, 1, 1.0d, 1, 0));
+ assertThrows(IllegalArgumentException.class,
+ () -> new LogFileReadLimits(0, 1, 1, 1.0d, 1, 1, 1));
+ assertThrows(IllegalArgumentException.class,
+ () -> new LogFileReadLimits(1, 1, 1, 1.0d, 1, 1, 0));
+ assertThrows(IllegalArgumentException.class,
+ () -> new LogFileReadLimits(1, 1, 1, 1.0d, 1, 1, 0, 1));
+ }
+
+ @Test
+ void exposesFiniteSecureDefaults() {
+ LogFileReadLimits defaults = LogFileReadLimits.defaults();
+
+ assertTrue(defaults.getMaxCompressedBytes() > 0);
+ assertTrue(defaults.getMaxExpandedBytes() > 0);
+ assertTrue(defaults.getMaxLineCharacters() > 0);
+ assertTrue(Double.isFinite(defaults.getMaxCompressionRatio()));
+ assertTrue(defaults.getCompressionRatioGraceBytes() > 0);
+ assertTrue(defaults.getMaxArchiveEntries() > 0);
+ assertTrue(defaults.getMaxArchiveMetadataBytes() > 0);
+ assertTrue(defaults.getMaxGzipHeaderBytes() > 0);
+ }
+
+ @Test
+ void enforcesExpandedBudgetAtomically() throws Exception {
+ LogFileReadBudget budget = new LogFileReadBudget(1000);
+ ExecutorService executor = Executors.newFixedThreadPool(4);
+ CountDownLatch start = new CountDownLatch(1);
+ try {
+ List> results = new ArrayList<>();
+ for (int index = 0; index < 4; index++) {
+ results.add(executor.submit(() -> {
+ start.await();
+ try {
+ budget.record(1000, temporaryDirectory.resolve("parallel.log"), null);
+ return true;
+ } catch (LogFileReadLimitExceededException expected) {
+ return false;
+ }
+ }));
+ }
+ start.countDown();
+ long successes = 0L;
+ for (Future result : results) {
+ if (result.get()) {
+ successes++;
+ }
+ }
+ assertEquals(1L, successes);
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
+ private static void assertLimit(
+ LogFileReadLimitExceededException.LimitType expected,
+ ThrowingOperation operation) {
+ LogFileReadLimitExceededException failure =
+ assertThrows(LogFileReadLimitExceededException.class, operation::run);
+ assertEquals(expected, failure.getLimitType());
+ }
+
+ private static String repeatedLines(int lineCount, int lineLength) {
+ return ("a".repeat(lineLength) + "\n").repeat(lineCount);
+ }
+
+ private static String timestampedLines(double timestamp, int lineCount, int lineLength) {
+ StringBuilder content = new StringBuilder(lineCount * (lineLength + 32));
+ for (int index = 0; index < lineCount; index++) {
+ content.append('[')
+ .append(timestamp + (index / 1000.0d))
+ .append("s][info][gc] ")
+ .append("a".repeat(lineLength))
+ .append('\n');
+ }
+ return content.toString();
+ }
+
+ private static String randomAsciiLine(int length) {
+ Random random = new Random(123456789L);
+ StringBuilder line = new StringBuilder(length + 1);
+ for (int index = 0; index < length; index++) {
+ line.append((char) ('!' + random.nextInt('~' - '!' + 1)));
+ }
+ return line.append('\n').toString();
+ }
+
+ private static void writeGzipMember(Path path, String content, boolean append) throws IOException {
+ StandardOpenOption[] options = append
+ ? new StandardOpenOption[] {StandardOpenOption.CREATE, StandardOpenOption.APPEND}
+ : new StandardOpenOption[] {StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING};
+ try (GZIPOutputStream gzip = new GZIPOutputStream(Files.newOutputStream(path, options))) {
+ gzip.write(content.getBytes(StandardCharsets.UTF_8));
+ }
+ }
+
+ private static void writeGzipWithHeaderCrc(
+ Path path,
+ String content,
+ boolean corruptChecksum) throws IOException {
+ Path ordinaryGzip = path.resolveSibling(path.getFileName() + ".ordinary");
+ writeGzipMember(ordinaryGzip, content, false);
+ byte[] ordinary = Files.readAllBytes(ordinaryGzip);
+ ordinary[3] |= 2;
+ CRC32 headerCrc = new CRC32();
+ headerCrc.update(ordinary, 0, 10);
+ int checksum = (int) headerCrc.getValue() & 0xffff;
+ if (corruptChecksum) {
+ checksum ^= 1;
+ }
+ byte[] withHeaderCrc = new byte[ordinary.length + 2];
+ System.arraycopy(ordinary, 0, withHeaderCrc, 0, 10);
+ withHeaderCrc[10] = (byte) checksum;
+ withHeaderCrc[11] = (byte) (checksum >>> 8);
+ System.arraycopy(ordinary, 10, withHeaderCrc, 12, ordinary.length - 10);
+ Files.write(path, withHeaderCrc);
+ Files.delete(ordinaryGzip);
+ }
+
+ private static void writeGzipWithFileName(
+ Path path,
+ String fileName,
+ String content) throws IOException {
+ Path ordinaryGzip = path.resolveSibling(path.getFileName() + ".ordinary");
+ writeGzipMember(ordinaryGzip, content, false);
+ byte[] ordinary = Files.readAllBytes(ordinaryGzip);
+ ordinary[3] |= 8;
+ byte[] name = fileName.getBytes(StandardCharsets.ISO_8859_1);
+ byte[] withName = new byte[ordinary.length + name.length + 1];
+ System.arraycopy(ordinary, 0, withName, 0, 10);
+ System.arraycopy(name, 0, withName, 10, name.length);
+ System.arraycopy(ordinary, 10, withName, 11 + name.length, ordinary.length - 10);
+ Files.write(path, withName);
+ Files.delete(ordinaryGzip);
+ }
+
+ private static void writeZip(Path path, Map entries) throws IOException {
+ try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(path))) {
+ for (Map.Entry entry : entries.entrySet()) {
+ zip.putNextEntry(new ZipEntry(entry.getKey()));
+ zip.write(entry.getValue().getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ }
+ }
+ }
+
+ private static byte[] withZip64EntryCount(byte[] ordinaryZip, long entryCount) {
+ int end = findSignature(ordinaryZip, 0x06054b50);
+ long directorySize = unsignedInt(ordinaryZip, end + 12);
+ long directoryOffset = unsignedInt(ordinaryZip, end + 16);
+ byte[] result = new byte[ordinaryZip.length + 76];
+ System.arraycopy(ordinaryZip, 0, result, 0, end);
+
+ int zip64End = end;
+ putUnsignedInt(result, zip64End, 0x06064b50L);
+ putLong(result, zip64End + 4, 44L);
+ putUnsignedShort(result, zip64End + 12, 45);
+ putUnsignedShort(result, zip64End + 14, 45);
+ putUnsignedInt(result, zip64End + 16, 0L);
+ putUnsignedInt(result, zip64End + 20, 0L);
+ putLong(result, zip64End + 24, entryCount);
+ putLong(result, zip64End + 32, entryCount);
+ putLong(result, zip64End + 40, directorySize);
+ putLong(result, zip64End + 48, directoryOffset);
+
+ int locator = zip64End + 56;
+ putUnsignedInt(result, locator, 0x07064b50L);
+ putUnsignedInt(result, locator + 4, 0L);
+ putLong(result, locator + 8, zip64End);
+ putUnsignedInt(result, locator + 16, 1L);
+
+ int newEnd = locator + 20;
+ System.arraycopy(ordinaryZip, end, result, newEnd, ordinaryZip.length - end);
+ putUnsignedShort(result, newEnd + 8, 0xffff);
+ putUnsignedShort(result, newEnd + 10, 0xffff);
+ return result;
+ }
+
+ private static byte[] withStoredDataDescriptor(byte[] ordinaryZip) {
+ int centralDirectory = findSignature(ordinaryZip, 0x02014b50);
+ int end = findSignature(ordinaryZip, 0x06054b50);
+ int localHeader = (int) unsignedInt(ordinaryZip, centralDirectory + 42);
+ int nameLength = Byte.toUnsignedInt(ordinaryZip[localHeader + 26])
+ | (Byte.toUnsignedInt(ordinaryZip[localHeader + 27]) << 8);
+ int extraLength = Byte.toUnsignedInt(ordinaryZip[localHeader + 28])
+ | (Byte.toUnsignedInt(ordinaryZip[localHeader + 29]) << 8);
+ int dataStart = localHeader + 30 + nameLength + extraLength;
+ int compressedSize = (int) unsignedInt(ordinaryZip, centralDirectory + 20);
+ int dataEnd = dataStart + compressedSize;
+ long crc = unsignedInt(ordinaryZip, centralDirectory + 16);
+ long expandedSize = unsignedInt(ordinaryZip, centralDirectory + 24);
+
+ byte[] result = new byte[ordinaryZip.length + 16];
+ System.arraycopy(ordinaryZip, 0, result, 0, dataEnd);
+ putUnsignedInt(result, dataEnd, 0x08074b50L);
+ putUnsignedInt(result, dataEnd + 4, crc);
+ putUnsignedInt(result, dataEnd + 8, compressedSize);
+ putUnsignedInt(result, dataEnd + 12, expandedSize);
+ System.arraycopy(
+ ordinaryZip,
+ dataEnd,
+ result,
+ dataEnd + 16,
+ ordinaryZip.length - dataEnd);
+
+ result[localHeader + 6] |= 8;
+ for (int offset = 14; offset < 26; offset++) {
+ result[localHeader + offset] = 0;
+ }
+ int shiftedCentralDirectory = centralDirectory + 16;
+ result[shiftedCentralDirectory + 8] |= 8;
+ int shiftedEnd = end + 16;
+ putUnsignedInt(result, shiftedEnd + 16, shiftedCentralDirectory);
+ return result;
+ }
+
+ private static byte[] zipWithArchiveExtraRecords(int recordCount) {
+ int directorySize = recordCount * 8;
+ byte[] archive = new byte[directorySize + 22];
+ for (int index = 0; index < recordCount; index++) {
+ putUnsignedInt(archive, index * 8, 0x08064b50L);
+ putUnsignedInt(archive, index * 8 + 4, 0L);
+ }
+ putUnsignedInt(archive, directorySize, 0x06054b50L);
+ putUnsignedInt(archive, directorySize + 12, directorySize);
+ putUnsignedInt(archive, directorySize + 16, 0L);
+ return archive;
+ }
+
+ private static int findSignature(byte[] bytes, int signature) {
+ for (int index = 0; index <= bytes.length - 4; index++) {
+ if ((int) unsignedInt(bytes, index) == signature) {
+ return index;
+ }
+ }
+ throw new IllegalArgumentException("ZIP signature not found");
+ }
+
+ private static long unsignedInt(byte[] bytes, int offset) {
+ return (long) Byte.toUnsignedInt(bytes[offset])
+ | ((long) Byte.toUnsignedInt(bytes[offset + 1]) << 8)
+ | ((long) Byte.toUnsignedInt(bytes[offset + 2]) << 16)
+ | ((long) Byte.toUnsignedInt(bytes[offset + 3]) << 24);
+ }
+
+ private static void putUnsignedShort(byte[] bytes, int offset, int value) {
+ bytes[offset] = (byte) value;
+ bytes[offset + 1] = (byte) (value >>> 8);
+ }
+
+ private static void putUnsignedInt(byte[] bytes, int offset, long value) {
+ bytes[offset] = (byte) value;
+ bytes[offset + 1] = (byte) (value >>> 8);
+ bytes[offset + 2] = (byte) (value >>> 16);
+ bytes[offset + 3] = (byte) (value >>> 24);
+ }
+
+ private static void putLong(byte[] bytes, int offset, long value) {
+ putUnsignedInt(bytes, offset, value);
+ putUnsignedInt(bytes, offset + 4, value >>> 32);
+ }
+
+ @FunctionalInterface
+ private interface ThrowingOperation {
+ void run() throws Exception;
+ }
+}
diff --git a/pom.xml b/pom.xml
index 1e69d2be5..3fea3f968 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,6 +46,7 @@
3.0.3-SNAPSHOT
13.10.0
+ 1.28.0
1.25.0
5.13.4
3.2.0
@@ -124,6 +125,11 @@
gctoolkit-gclogs
${project.version}