From cce7cc10b3afceb44d8249e2d8f7df0a374be84f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 05:28:53 +0000 Subject: [PATCH 001/146] fix: read a stream's data each time it is asked for, and keep none of it A stream read from a file kept a copy of its raw data for as long as the stream was held, by the object cache or by the caller. Streams whose data share a stretch of the file each kept their own: two hundred of them held 200 MB from a file of 1 MB, and validating three hundred nested object streams held 300 MB, whatever the object stream budget let go. The data is read each time it is asked for now, and kept by nothing: a source in memory lends it where it lies, and any other reads it into an array of its length, handed back whole on a short read rather than copied again. Refs #170 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 29 ++++-- src/AdCodicem.Pdf/IO/PdfFileSource.cs | 15 +++ .../DocumentReaderTests.cs | 29 ++++++ .../HostileInputMemoryTests.cs | 71 ++++++++++++++ .../AdCodicem.Pdf.Tests/HostileInputTests.cs | 95 +++++++++++++++++++ 5 files changed, 229 insertions(+), 10 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 48aa5af9..60ec289a 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -175,11 +175,10 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// parsing of the object it loads besides. No level holds decoded data while it loads the next: a stream resolves /// its filters and every parameter they read before its first filter runs (), so /// that a chain of loads holds at once one stream's decoding, and the object streams - /// keeps. One thing remains: each object stream loaded on the chain stays in the object - /// cache with the raw data it read, until the cache lets it go (#170). A stream an object stream holds, which ISO - /// 32000-1 (7.5.7) forbids, is no exception: it is no value, and its /Length is not read (#161). The count - /// alone does not bound the stack, since each load parses as deep as the parser allows: the stack is asked as well - /// before each load. + /// keeps; no stream keeps the raw data it read (#170). A stream an object stream + /// holds, which ISO 32000-1 (7.5.7) forbids, is no exception: it is no value, and its /Length is not read + /// (#161). The count alone does not bound the stack, since each load parses as deep as the parser allows: the stack + /// is asked as well before each load. /// private const int MaxNestedLoads = 64; @@ -5875,9 +5874,18 @@ private void PreserveChainIndex() /// A stream's data in the file, read the first time something asks for it, and refused, read or not, once the /// document is disposed. /// + /// + /// A stream's data in the file: where it lies and how long it is, read each time it is asked for, and kept by + /// nothing (#170). + /// + /// + /// Streams whose data share a stretch of the file — overlapping objects —, or the many streams of a document read once + /// each, hold none of it: what a caller keeps is its own, and memory follows what is being decoded. A source in + /// memory lends the bytes where they lie, without a copy, valid for as long as its memory is; any other reads them + /// into an array of their length. + /// private sealed class FileStreamData(PdfFileReader reader, long offset, int length, PdfLimitGuard guard) : PdfStreamData { - private byte[]? _bytes; private bool _cutByGuard; public override int Length => length; @@ -5895,15 +5903,16 @@ public override ReadOnlyMemory GetBytes() { ObjectDisposedException.ThrowIf(reader._closed, typeof(PdfDocument)); - if (_bytes is not null) + if (reader._source.TryLend(offset, length, out var lent)) { - return _bytes; + return lent; } + // A source that gives less than it holds leaves the array longer than what it gave: the bytes it gave are + // handed back, not copied again. var buffer = new byte[length]; var read = reader._source.Read(offset, buffer); - _bytes = read == length ? buffer : buffer[..read]; - return _bytes; + return buffer.AsMemory(0, read); } } diff --git a/src/AdCodicem.Pdf/IO/PdfFileSource.cs b/src/AdCodicem.Pdf/IO/PdfFileSource.cs index 04aa2331..35c18077 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileSource.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileSource.cs @@ -94,6 +94,21 @@ private protected virtual bool TryGetMemory(long offset, int length, out ReadOnl return false; } + /// + /// Lends the bytes at where the source holds them, without a copy; + /// false for a source that reads them from a file, or bytes it does not hold. + /// + internal bool TryLend(long offset, int length, out ReadOnlyMemory memory) + { + if (offset < 0 || length < 0 || offset > Length - length) + { + memory = default; + return false; + } + + return TryGetMemory(offset, length, out memory); + } + /// /// Returns a view of bytes at , clamped to the file. /// The window must be disposed; for a file-backed source it holds a pooled buffer. diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index f69018d4..db331c6d 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -676,6 +676,35 @@ public void A_source_gives_an_empty_window_for_a_range_outside_itself_or_of_no_l window.Offset.Should().Be(0); } + [Fact] + public void A_stream_s_data_is_read_from_the_file_each_time_it_is_asked_for_and_kept_by_nothing() + { + // A stream's data was read once and kept with the stream for as long as the stream was held, by the object cache or + // by the caller (#170). It is read each time it is asked for: a file in memory lends it where it lies, and any other + // source is read again. + var file = new TestPdfBuilder() + .WithObject(1, "<< /Type /Catalog /Pages 2 0 R >>") + .WithObject(2, "<< /Type /Pages /Kids [] /Count 0 >>") + .Stream(3, string.Empty, new string('A', 20_000)) + .BuildClassic(rootNumber: 1); + using var source = new StrictCountingSource(file); + using var document = PdfDocument.Open(source, options: null, ownsSource: false); + var stream = document.GetObject(new PdfObjectId(3)).AsStream().Required(); + var before = source.BytesRead; + + var first = stream.GetRawBytes().ToArray(); + var second = stream.GetRawBytes().ToArray(); + + first.Should().Equal(second); + first.Should().HaveCount(20_000).And.OnlyContain(value => value == (byte)'A'); + (source.BytesRead - before).Should().Be(40_000, "the data is read each time it is asked for"); + + using var inMemory = PdfDocument.Open(file); + var lent = inMemory.GetObject(new PdfObjectId(3)).AsStream().Required().GetRawBytes(); + System.Runtime.InteropServices.MemoryMarshal.TryGetArray(lent, out var segment).Should().BeTrue(); + segment.Array.Should().BeSameAs(file, "a file in memory lends the data where it lies"); + } + [Fact] public void Disposing_a_document_twice_is_harmless() { diff --git a/tests/AdCodicem.Pdf.Tests/HostileInputMemoryTests.cs b/tests/AdCodicem.Pdf.Tests/HostileInputMemoryTests.cs index 5d8ddf9d..1de290e4 100644 --- a/tests/AdCodicem.Pdf.Tests/HostileInputMemoryTests.cs +++ b/tests/AdCodicem.Pdf.Tests/HostileInputMemoryTests.cs @@ -5,6 +5,7 @@ using AdCodicem.Pdf.Documents; using AdCodicem.Pdf.IO; using AdCodicem.Pdf.Objects; +using AdCodicem.Pdf.Validation; namespace AdCodicem.Pdf.Tests; @@ -70,6 +71,76 @@ public void Disposing_a_document_lets_go_of_the_object_streams_it_decoded_though GC.KeepAlive(held); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Streams_whose_data_share_a_stretch_of_the_file_hold_none_of_it_once_read(bool fromPath) + { + // Two hundred streams, each header inside the data of the one before, every /Length confirmed by the one endstream + // a megabyte on: each stream's data is the whole stretch, and each kept a copy of it for as long as the stream was + // held, 200 MB from a file of 1 MB (#170). The streams are held here; their data is not. + const int Streams = 200; + var file = HostileInputTests.Overlapping(Streams, filler: 1024 * 1024); + var path = fromPath ? Path.GetTempFileName() : null; + + try + { + if (path is not null) + { + File.WriteAllBytes(path, file); + } + + var before = Retained(); + using var document = path is null ? PdfDocument.Open(file) : PdfDocument.Open(path); + var held = new List(Streams); + long read = 0; + + for (var k = 0; k < Streams; k++) + { + var stream = document.GetObject(new PdfObjectId(3 + k)).AsStream().Required(); + read += stream.GetRawBytes().Length; + held.Add(stream); + } + + var after = Retained(); + + read.Should().BeGreaterThan(Streams * 1024L * 1024, "each stream's data is the whole stretch"); + after.Should().BeLessThan(before + (8L * 1024 * 1024), "the streams, and no copy of their data"); + document.Diagnostics.Should().BeEmpty(); + GC.KeepAlive(held); + } + finally + { + if (path is not null) + { + File.Delete(path); + } + } + } + + [Fact] + public void Validating_object_streams_whose_data_share_a_stretch_of_the_file_holds_no_copy_of_their_data() + { + // Three hundred object streams of one member each, each nested in the one before, every /Length confirmed by the one + // endstream a megabyte on: validating them, and reading their members, kept each one's data with its stream in the + // object cache, whatever the object stream budget let go — 300 MB from a file of 1 MB (#170). + const int Streams = 300; + var file = HostileInputTests.Overlapping(Streams, filler: 1024 * 1024, objectStreams: true); + var before = Retained(); + using var document = PdfDocument.Open(file); + + new PdfValidator().Validate(document); + + for (var k = 0; k < Streams; k++) + { + document.GetObject(new PdfObjectId(3000 + k)); + } + + var after = Retained(); + + after.Should().BeLessThan(before + (8L * 1024 * 1024), "a file in memory lends each object stream its data, and no copy is kept"); + } + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] private static (PdfObject Held, long Kept) OpenDecodeAndDispose(byte[] file) { diff --git a/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs b/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs index b56daab9..a532bae0 100644 --- a/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs +++ b/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs @@ -169,6 +169,37 @@ public void An_object_stream_header_s_tables_start_no_larger_than_its_bytes_can_ allocated.Should().BeLessThanOrEqualTo(2 * (sizeof(int) + 32), "two tables of one pair each"); } + [Fact] + public void Reading_streams_whose_data_share_a_stretch_of_a_file_held_in_memory_allocates_none_of_their_data() + { + // Two hundred streams, each header inside the data of the one before, every /Length confirmed by the one endstream + // a megabyte on: each stream's data is the whole stretch, and reading each one's copied it, 200 MB from a file of + // 1 MB (#170). A source in memory lends each stream's data where it lies. + const int Streams = 200; + var file = Overlapping(Streams, filler: 1024 * 1024); + + var allocated = Allocations.Least(() => PdfDocument.Open(file), ReadEvery, out var document, out var read); + + using (document) + { + read.Should().BeGreaterThan(Streams * 1024L * 1024, "each stream's data is the whole stretch"); + allocated.Should().BeLessThan(2L * 1024 * 1024, "the streams' objects, and none of their data"); + document.Diagnostics.Should().BeEmpty(); + } + + static long ReadEvery(PdfDocument document) + { + long read = 0; + + for (var k = 0; k < Streams; k++) + { + read += document.GetObject(new PdfObjectId(3 + k)).AsStream().Required().GetRawBytes().Length; + } + + return read; + } + } + [Fact] public void An_object_stream_header_s_tables_grow_no_larger_than_its_bytes_can_hold() { @@ -3464,6 +3495,70 @@ private static byte[] Claiming(int count) return file.Bytes; } + /// + /// Writes a file of streams, objects 3 and on, each one's header inside the data of the one + /// before, every /Length ending at the one endstream after bytes: each length + /// confirmed, each stream's data the whole stretch from its start to that endstream (#170). With + /// , objects 10 and on, each an object stream of one member, objects 3000 and on, that a + /// cross-reference stream places. + /// + internal static byte[] Overlapping(int streams, int filler, bool objectStreams = false) + { + var file = new HandWrittenFile(version: "1.5"); + var rows = new Dictionary + { + [1] = (1, file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), 0), + [2] = (1, file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), 0), + }; + var start = file.Position; + var body = new StringBuilder(); + var lengthAt = new int[streams]; + var dataStart = new int[streams]; + + for (var k = 0; k < streams; k++) + { + var number = (objectStreams ? 10 : 3) + k; + var member = string.Create(CultureInfo.InvariantCulture, $"{3000 + k} 0 "); + rows[number] = (1, start + body.Length, 0); + body.Append(CultureInfo.InvariantCulture, $"{number} 0 obj\n<< "); + + if (objectStreams) + { + body.Append(CultureInfo.InvariantCulture, $"/Type /ObjStm /N 1 /First {member.Length} "); + } + + body.Append("/Length "); + lengthAt[k] = body.Length; + body.Append("0000000000 >>\nstream\n"); + dataStart[k] = body.Length; + + if (objectStreams) + { + body.Append(member); + rows[3000 + k] = (2, number, 0); + } + } + + body.Append('x', filler); + var end = body.Length; + body.Append("\nendstream\nendobj\n"); + + for (var k = 0; k < streams; k++) + { + var digits = (end - dataStart[k]).ToString("D10", CultureInfo.InvariantCulture); + + for (var d = 0; d < digits.Length; d++) + { + body[lengthAt[k] + d] = digits[d]; + } + } + + file.Raw(body.ToString()); + var index = objectStreams ? 6000 : 1000; + file.CrossReferenceStream(index, index + 1, rows, "/Root 1 0 R"); + return file.Bytes; + } + /// /// Writes a file whose object stream holds members one byte apart, objects 1000 and on, over /// as many open brackets: read to the end of the data, each would take every bracket after it (#200). From 5dd820615c5f624f7c9647abb78ce2b592f1e8c5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 05:31:05 +0000 Subject: [PATCH 002/146] feat: bound the stream data the reader reads at once by MaxStreamLength A stream's encoded data was bounded by the file alone. Decoding handed back data no filter decodes - none named, an image filter, one the library does not know, Flate over data that is none - whole, past MaxDecodedStreamLength and unreported, and a /Length between what an array holds and what an int does threw an OutOfMemoryException each time the data was read. PdfReaderLimits.MaxStreamLength bounds what is read of one stream at once, 256 MB by default, under the code limit.stream-length: past it the data is cut, marked cut by a guard, so that what the filters lose past the cut is no damage, and the guard is reported once for each stream, or thrown under ThrowOnLimit. Data decoding hands back as it is falls under it, and MaxDecodedStreamLength keeps bounding what filters produce. A length past what an array holds ends at the guard, whose ceiling is the array's. The stream's length stays the one the file holds. The manifest can raise the new limit. Refs #170 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../Diagnostics/PdfDiagnosticCodes.cs | 8 + .../Documents/PdfReaderLimits.cs | 22 ++ src/AdCodicem.Pdf/IO/PdfFileReader.cs | 23 ++- src/AdCodicem.Pdf/IO/PdfLimit.cs | 3 + src/AdCodicem.Pdf/IO/PdfLimitGuard.cs | 3 + .../CorpusOptions.cs | 1 + tests/AdCodicem.Pdf.TestSupport/Corpus.cs | 3 + .../AdCodicem.Pdf.Tests/CorpusReadingTests.cs | 2 + .../AdCodicem.Pdf.Tests/ReaderLimitsTests.cs | 188 +++++++++++++++++- tests/corpus/manifest.schema.json | 7 + 10 files changed, 256 insertions(+), 4 deletions(-) diff --git a/src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs b/src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs index a83b6148..a16805c7 100644 --- a/src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs +++ b/src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs @@ -218,6 +218,14 @@ public static class PdfDiagnosticCodes /// public const string LimitDecodedStream = "limit.decoded-stream"; + /// + /// A stream's encoded data runs past , and was read up to it: + /// what lies within it is decoded as data the reader cut, and handed back as it is where decoding hands its data back + /// undecoded. Reported once for each stream, when its data is first read. Like every limit.* code, the guard is + /// the reader's, not damage in the file. + /// + public const string LimitStreamLength = "limit.stream-length"; + /// /// An object is longer than , and only the part /// within it was parsed. An object an object stream holds is bounded alike, and reported once for each definition of diff --git a/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs b/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs index afb699bb..a8a8721c 100644 --- a/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs +++ b/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs @@ -33,6 +33,7 @@ public sealed record PdfReaderLimits public static PdfReaderLimits Unbounded { get; } = new() { MaxDecodedStreamLength = int.MaxValue, + MaxStreamLength = int.MaxValue, MaxObjectLength = int.MaxValue, MaxXRefSectionLength = int.MaxValue, MaxXRefSectionCount = int.MaxValue, @@ -61,6 +62,27 @@ public sealed record PdfReaderLimits /// The value is zero or less. public int MaxDecodedStreamLength { get; init => field = Length(value); } = 256 * 1024 * 1024; + /// + /// Gets the most of one stream's encoded data the reader reads at once, in bytes. Defaults to 256 MB: a /Length + /// of a few digits can claim gigabytes, and a scanned map can hold an uncompressed image of 300 MB. + /// + /// + /// + /// The encoded data is read in one piece each time it is asked for — , or + /// decoding it —, so a value above is taken as that length, and a /Length past it + /// is cut there. Data past the bound is cut at it: what lies within it is read, and decoded as data the reader cut, + /// whose lost tail is no damage. Reaching it reports , + /// once for each stream. stays the length the file holds. + /// + /// + /// Data that decoding hands back as it is — no filter, an image filter, a filter the library does not know, one that + /// could not decode it — is bounded by this guard, not by , which bounds what the + /// filters produce: a large uncompressed image needs this bound raised alone (#170). + /// + /// + /// The value is zero or less. + public int MaxStreamLength { get; init => field = Length(value); } = 256 * 1024 * 1024; + /// /// Gets the longest object the reader parses, in bytes, the data of a stream aside, whether the file writes it /// directly or an object stream holds it. Defaults to 16 MB, which a direct array of a million references would need. diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 60ec289a..4c9def51 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -5882,7 +5882,8 @@ private void PreserveChainIndex() /// Streams whose data share a stretch of the file — overlapping objects —, or the many streams of a document read once /// each, hold none of it: what a caller keeps is its own, and memory follows what is being decoded. A source in /// memory lends the bytes where they lie, without a copy, valid for as long as its memory is; any other reads them - /// into an array of their length. + /// into an array of their length. No more than bytes are read: past it, + /// the data is cut there, marked cut by a guard, and the guard reported, once for each stream. /// private sealed class FileStreamData(PdfFileReader reader, long offset, int length, PdfLimitGuard guard) : PdfStreamData { @@ -5899,18 +5900,34 @@ private sealed class FileStreamData(PdfFileReader reader, long offset, int lengt public void MarkCutByGuard() => _cutByGuard = true; /// The document was disposed. + /// + /// The data runs past , and the document was opened to throw when it does. + /// public override ReadOnlyMemory GetBytes() { ObjectDisposedException.ThrowIf(reader._closed, typeof(PdfDocument)); - if (reader._source.TryLend(offset, length, out var lent)) + var count = length; + var bound = guard.Bound(PdfLimit.StreamLength); + + if (count > bound) + { + _cutByGuard = true; + count = bound; + reader.ReachLimit( + PdfLimit.StreamLength, + $"A stream's data runs past {PdfLimitGuard.FormatLength(bound)}; only what lies within it was read.", + offset); + } + + if (reader._source.TryLend(offset, count, out var lent)) { return lent; } // A source that gives less than it holds leaves the array longer than what it gave: the bytes it gave are // handed back, not copied again. - var buffer = new byte[length]; + var buffer = new byte[count]; var read = reader._source.Read(offset, buffer); return buffer.AsMemory(0, read); } diff --git a/src/AdCodicem.Pdf/IO/PdfLimit.cs b/src/AdCodicem.Pdf/IO/PdfLimit.cs index 7968a0ee..2aed807d 100644 --- a/src/AdCodicem.Pdf/IO/PdfLimit.cs +++ b/src/AdCodicem.Pdf/IO/PdfLimit.cs @@ -9,6 +9,9 @@ internal enum PdfLimit /// What one stream may decode to. DecodedStream, + /// How much of one stream's encoded data may be read at once. + StreamLength, + /// How long an object may be, the data of a stream aside. Object, diff --git a/src/AdCodicem.Pdf/IO/PdfLimitGuard.cs b/src/AdCodicem.Pdf/IO/PdfLimitGuard.cs index 32e003d9..3daaf962 100644 --- a/src/AdCodicem.Pdf/IO/PdfLimitGuard.cs +++ b/src/AdCodicem.Pdf/IO/PdfLimitGuard.cs @@ -36,6 +36,7 @@ internal sealed class PdfLimitGuard(PdfReaderLimits limits, bool throwOnLimit, P public int Bound(PdfLimit limit) => limit switch { PdfLimit.DecodedStream => limits.MaxDecodedStreamLength, + PdfLimit.StreamLength => limits.MaxStreamLength, PdfLimit.Object => limits.MaxObjectLength, PdfLimit.XRefSectionLength => limits.MaxXRefSectionLength, PdfLimit.XRefSectionCount => limits.MaxXRefSectionCount, @@ -78,6 +79,7 @@ public void Reach(PdfLimit limit, PdfDiagnostics? diagnostics, string what, long private static string Code(PdfLimit limit) => limit switch { PdfLimit.DecodedStream => PdfDiagnosticCodes.LimitDecodedStream, + PdfLimit.StreamLength => PdfDiagnosticCodes.LimitStreamLength, PdfLimit.Object => PdfDiagnosticCodes.LimitObject, PdfLimit.XRefSectionLength => PdfDiagnosticCodes.LimitXRefSectionLength, PdfLimit.XRefSectionCount => PdfDiagnosticCodes.LimitXRefSectionCount, @@ -89,6 +91,7 @@ public void Reach(PdfLimit limit, PdfDiagnostics? diagnostics, string what, long private static string Name(PdfLimit limit) => limit switch { PdfLimit.DecodedStream => nameof(PdfReaderLimits.MaxDecodedStreamLength), + PdfLimit.StreamLength => nameof(PdfReaderLimits.MaxStreamLength), PdfLimit.Object => nameof(PdfReaderLimits.MaxObjectLength), PdfLimit.XRefSectionLength => nameof(PdfReaderLimits.MaxXRefSectionLength), PdfLimit.XRefSectionCount => nameof(PdfReaderLimits.MaxXRefSectionCount), diff --git a/tests/AdCodicem.Pdf.IntegrationTests/CorpusOptions.cs b/tests/AdCodicem.Pdf.IntegrationTests/CorpusOptions.cs index a3253f01..36f3e2d7 100644 --- a/tests/AdCodicem.Pdf.IntegrationTests/CorpusOptions.cs +++ b/tests/AdCodicem.Pdf.IntegrationTests/CorpusOptions.cs @@ -18,6 +18,7 @@ public static PdfReaderOptions For(CorpusDocument entry) limits = limits with { MaxDecodedStreamLength = raised.MaxDecodedStreamLength ?? limits.MaxDecodedStreamLength, + MaxStreamLength = raised.MaxStreamLength ?? limits.MaxStreamLength, MaxObjectLength = raised.MaxObjectLength ?? limits.MaxObjectLength, MaxXRefSectionLength = raised.MaxXRefSectionLength ?? limits.MaxXRefSectionLength, MaxXRefSectionCount = raised.MaxXRefSectionCount ?? limits.MaxXRefSectionCount, diff --git a/tests/AdCodicem.Pdf.TestSupport/Corpus.cs b/tests/AdCodicem.Pdf.TestSupport/Corpus.cs index 5957349f..c1158844 100644 --- a/tests/AdCodicem.Pdf.TestSupport/Corpus.cs +++ b/tests/AdCodicem.Pdf.TestSupport/Corpus.cs @@ -110,6 +110,9 @@ public sealed record CorpusReaderLimits /// What one stream may decode to, in bytes. public int? MaxDecodedStreamLength { get; init; } + /// The most of one stream's encoded data read at once, in bytes. + public int? MaxStreamLength { get; init; } + /// The longest object, the data of a stream aside, in bytes. public int? MaxObjectLength { get; init; } diff --git a/tests/AdCodicem.Pdf.Tests/CorpusReadingTests.cs b/tests/AdCodicem.Pdf.Tests/CorpusReadingTests.cs index 3d50360a..2dd3e20e 100644 --- a/tests/AdCodicem.Pdf.Tests/CorpusReadingTests.cs +++ b/tests/AdCodicem.Pdf.Tests/CorpusReadingTests.cs @@ -528,6 +528,7 @@ private static PdfReaderLimits LimitsFor(CorpusReaderLimits raised) return limits with { MaxDecodedStreamLength = raised.MaxDecodedStreamLength ?? limits.MaxDecodedStreamLength, + MaxStreamLength = raised.MaxStreamLength ?? limits.MaxStreamLength, MaxObjectLength = raised.MaxObjectLength ?? limits.MaxObjectLength, MaxXRefSectionLength = raised.MaxXRefSectionLength ?? limits.MaxXRefSectionLength, MaxXRefSectionCount = raised.MaxXRefSectionCount ?? limits.MaxXRefSectionCount, @@ -542,6 +543,7 @@ private static bool NamesARaisedLimit(string message, PdfReaderLimits raised) var defaults = PdfReaderLimits.Default; return (raised.MaxDecodedStreamLength != defaults.MaxDecodedStreamLength && message.Contains("PdfReaderLimits.MaxDecodedStreamLength", StringComparison.Ordinal)) + || (raised.MaxStreamLength != defaults.MaxStreamLength && message.Contains("PdfReaderLimits.MaxStreamLength", StringComparison.Ordinal)) || (raised.MaxObjectLength != defaults.MaxObjectLength && message.Contains("PdfReaderLimits.MaxObjectLength", StringComparison.Ordinal)) || (raised.MaxXRefSectionLength != defaults.MaxXRefSectionLength && message.Contains("PdfReaderLimits.MaxXRefSectionLength", StringComparison.Ordinal)) || (raised.MaxXRefSectionCount != defaults.MaxXRefSectionCount && message.Contains("PdfReaderLimits.MaxXRefSectionCount", StringComparison.Ordinal)) diff --git a/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs b/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs index dc1b152e..374cb54b 100644 --- a/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs @@ -22,7 +22,7 @@ public class ReaderLimitsTests /// A guard for each bound of , the trailer's reached two ways. public static TheoryData Guards => - ["decoded stream", "object", "section length", "section count", "trailer", "cross-reference stream", "index entries", "filter chain"]; + ["decoded stream", "stream length", "object", "section length", "section count", "trailer", "cross-reference stream", "index entries", "filter chain"]; [Fact] public void The_defaults_are_the_bounds_the_reader_kept_before_they_were_options() @@ -30,6 +30,7 @@ public void The_defaults_are_the_bounds_the_reader_kept_before_they_were_options var limits = PdfReaderLimits.Default; limits.MaxDecodedStreamLength.Should().Be(256 * 1024 * 1024); + limits.MaxStreamLength.Should().Be(256 * 1024 * 1024); limits.MaxObjectLength.Should().Be(16 * 1024 * 1024); limits.MaxXRefSectionLength.Should().Be(64 * 1024 * 1024); limits.MaxXRefSectionCount.Should().Be(1024); @@ -46,6 +47,7 @@ public void Unbounded_takes_every_guard_to_the_most_the_implementation_holds() var limits = PdfReaderLimits.Unbounded; limits.MaxDecodedStreamLength.Should().Be(Array.MaxLength); + limits.MaxStreamLength.Should().Be(Array.MaxLength); limits.MaxObjectLength.Should().Be(Array.MaxLength); limits.MaxXRefSectionLength.Should().Be(Array.MaxLength); limits.MaxXRefSectionCount.Should().Be(int.MaxValue); @@ -56,6 +58,7 @@ public void Unbounded_takes_every_guard_to_the_most_the_implementation_holds() [Theory] [InlineData(nameof(PdfReaderLimits.MaxDecodedStreamLength), 0)] + [InlineData(nameof(PdfReaderLimits.MaxStreamLength), 0)] [InlineData(nameof(PdfReaderLimits.MaxObjectLength), -1)] [InlineData(nameof(PdfReaderLimits.MaxXRefSectionLength), int.MinValue)] [InlineData(nameof(PdfReaderLimits.MaxXRefSectionCount), 0)] @@ -75,6 +78,7 @@ public void A_length_beyond_what_an_array_holds_is_taken_as_that_length() var limits = PdfReaderLimits.Default with { MaxDecodedStreamLength = int.MaxValue, + MaxStreamLength = int.MaxValue, MaxObjectLength = int.MaxValue, MaxXRefSectionLength = int.MaxValue, MaxXRefSectionCount = int.MaxValue, @@ -112,6 +116,7 @@ PdfReaderLimits.Default with throwOnLimit: false); unbounded.Reach(PdfLimit.DecodedStream, diagnostics, "Cut.", 7); + unbounded.Reach(PdfLimit.StreamLength, diagnostics, "Cut.", 7); unbounded.Reach(PdfLimit.XRefSectionCount, diagnostics, "Cut.", 8); justBelow.Reach(PdfLimit.XRefSectionCount, diagnostics, "Cut.", 9); unbounded.Reach(PdfLimit.IndexEntries, diagnostics, "Cut.", 10); @@ -121,6 +126,7 @@ PdfReaderLimits.Default with diagnostics.Select(entry => entry.Message).Should().Equal( "Cut. PdfReaderLimits.MaxDecodedStreamLength is already at the most the reader can hold.", + "Cut. PdfReaderLimits.MaxStreamLength is already at the most the reader can hold.", "Cut. PdfReaderLimits.MaxXRefSectionCount is already at the most the reader can hold.", "Cut. Raise PdfReaderLimits.MaxXRefSectionCount to read past it.", "Cut. PdfReaderLimits.MaxIndexEntries is already at the most the reader can hold.", @@ -234,6 +240,95 @@ public void A_stream_decoded_without_diagnostics_reports_the_bound_to_its_docume document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitDecodedStream); } + [Fact] + public void A_stream_cut_at_MaxStreamLength_decodes_what_lies_within_it_as_data_the_reader_cut_and_is_reported_once() + { + // A hundred kilobytes of Flate data, which a bound of 10 KB cuts: the data is read up to the bound however often it + // is asked for, its length stays the file's, and what the Flate data lacks past the cut is the guard's, not damage + // in the file (#170). + var compressed = HostileInputTests.Compress(RandomBytes(100_000)); + var written = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = written.Object(1, Catalog), + [2] = written.Object(2, Pages), + [3] = written.Object(3, string.Create(CultureInfo.InvariantCulture, $"<< /Filter /FlateDecode /Length {compressed.Length} >>\nstream\n{Encoding.Latin1.GetString(compressed)}\nendstream")), + }; + written.Table(4, rows, "/Size 4 /Root 1 0 R"); + var file = written.Bytes; + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxStreamLength = 10 * 1024 } }; + using var document = PdfDocument.Open(file, options); + var stream = document.GetObject(new PdfObjectId(3)).AsStream().Required(); + + stream.GetRawBytes().Length.Should().Be(10 * 1024); + stream.GetRawBytes().ToArray().Should().Equal(compressed.AsSpan(0, 10 * 1024).ToArray()); + stream.Data.Length.Should().Be(compressed.Length, "the length is the file's"); + var decoded = stream.Decode(document.Diagnostics); + + decoded.Length.Should().BeGreaterThan(0).And.BeLessThan(100_000); + var report = document.Diagnostics.Should().ContainSingle().Which; + report.Code.Should().Be(PdfDiagnosticCodes.LimitStreamLength); + report.Message.Should().Be("A stream's data runs past 10 KB; only what lies within it was read. Raise PdfReaderLimits.MaxStreamLength to read past it."); + report.Position.Should().Be(stream.Data.Position); + } + + [Theory] + [InlineData("")] + [InlineData("/Filter 5")] + [InlineData("/Filter []")] + [InlineData("/Filter [null]")] + [InlineData("/Filter /DCTDecode")] + [InlineData("/Filter /JPXDecode")] + [InlineData("/Filter /JBIG2Decode")] + [InlineData("/Filter /CCITTFaxDecode")] + [InlineData("/Filter /FlateDecode")] + [InlineData("/Filter /Crypt")] + [InlineData("/Filter /Foo")] + [InlineData("/Filter [/Foo /FlateDecode]")] + public void Data_decoding_hands_back_as_it_is_is_bounded_by_MaxStreamLength_and_not_by_MaxDecodedStreamLength(string filter) + { + // Fifty thousand bytes that no filter decodes — none named, an image filter, one the library does not know, Flate + // over data that is none —: decoding hands them back as they are, whole under a decoded bound of 1 KB, unreported. + // Their bound is the one on what is read, here 10 KB, which a large uncompressed image raises alone (#170). + var file = new TestPdfBuilder() + .WithObject(1, Catalog) + .WithObject(2, Pages) + .Stream(3, filter, new string('4', 50_000)) + .BuildClassic(rootNumber: 1); + var limits = PdfReaderLimits.Default with { MaxStreamLength = 10 * 1024, MaxDecodedStreamLength = 1024 }; + using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = limits }); + var stream = document.GetObject(new PdfObjectId(3)).AsStream().Required(); + + var decoded = stream.Decode(document.Diagnostics); + + decoded.Length.Should().Be(10 * 1024); + document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.LimitStreamLength); + document.Diagnostics.Should().NotContain(entry => entry.Code == PdfDiagnosticCodes.LimitDecodedStream); + } + + [Theory] + [InlineData(2_147_483_600L)] + [InlineData(2_147_483_647L)] + public void A_length_past_what_an_array_holds_ends_at_the_guard_and_never_in_an_exhausted_memory(long declared) + { + // A file of about 2 GB whose stream declares a length between the most an array holds and the most an int does, + // which the file holds: reading its data threw an OutOfMemoryException each time, and reported nothing (#170). Under + // a bound of 1 MB, a megabyte is read and the guard reported; under no bound the guard's ceiling is the array's. + using var source = SparseSource.Huge(declared); + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxStreamLength = 1024 * 1024 } }; + using var document = PdfDocument.Open(source, options, ownsSource: false); + var stream = document.GetObject(new PdfObjectId(3)).AsStream().Required(); + + stream.RawLength.Should().Be((int)declared); + stream.GetRawBytes().Length.Should().Be(1024 * 1024); + stream.Decode(document.Diagnostics).Length.Should().Be(1024 * 1024); + document.Diagnostics.Should().ContainSingle().Which.Code.Should().Be(PdfDiagnosticCodes.LimitStreamLength); + + using var throwing = PdfDocument.Open(source, options with { ThrowOnLimit = true }, ownsSource: false); + var reading = () => throwing.GetObject(new PdfObjectId(3)).AsStream().Required().GetRawBytes(); + reading.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxStreamLength)); + } + [Fact] public void An_object_read_again_after_the_cache_let_it_go_is_reported_once_and_throws_each_time() { @@ -3150,9 +3245,75 @@ private static PdfReaderOptions TrailerCut() => private static string LongArray { get; } = "[" + string.Join(' ', Enumerable.Range(0, 2000)) + "]"; + private static byte[] RandomBytes(int length) + { + var bytes = new byte[length]; + new Random(170).NextBytes(bytes); + return bytes; + } + + /// + /// A source of about 2 GB that holds none of it: a head, white space, and a tail at its end, read as the file would + /// be, so that a stream may declare a length no array holds. + /// + private sealed class SparseSource(byte[] head, long tailAt, byte[] tail) : PdfFileSource + { + public override long Length => tailAt + tail.Length; + + /// + /// A file whose object 3 is a stream of bytes, which an endstream after them + /// confirms, its table and trailer past them. + /// + public static SparseSource Huge(long declared) + { + var text = string.Create( + CultureInfo.InvariantCulture, + $"%PDF-1.7\n1 0 obj\n{Catalog}\nendobj\n2 0 obj\n{Pages}\nendobj\n3 0 obj\n<< /Length {declared} >>\nstream\n"); + var head = Encoding.Latin1.GetBytes(text); + var tailAt = head.Length + declared; + var xref = tailAt + "\nendstream\nendobj\n".Length; + var tail = Encoding.Latin1.GetBytes(string.Create( + CultureInfo.InvariantCulture, + $"\nendstream\nendobj\nxref\n0 4\n0000000000 65535 f\r\n{9:D10} 00000 n\r\n{text.IndexOf("2 0 obj", StringComparison.Ordinal):D10} 00000 n\r\n{text.IndexOf("3 0 obj", StringComparison.Ordinal):D10} 00000 n\r\ntrailer\n<< /Size 4 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n")); + return new SparseSource(head, tailAt, tail); + } + + public override int Read(long offset, Span buffer) + { + var count = (int)Math.Clamp(Length - offset, 0, buffer.Length); + + for (var i = 0; i < count;) + { + var at = offset + i; + int run; + + if (at < head.Length) + { + run = (int)Math.Min(count - i, head.Length - at); + head.AsSpan((int)at, run).CopyTo(buffer[i..]); + } + else if (at < tailAt) + { + run = (int)Math.Min(count - i, tailAt - at); + buffer.Slice(i, run).Fill((byte)' '); + } + else + { + run = (int)Math.Min(count - i, tail.Length - (at - tailAt)); + tail.AsSpan((int)(at - tailAt), run).CopyTo(buffer[i..]); + } + + i += run; + } + + return count; + } + } + private static PdfReaderLimits With(PdfReaderLimits limits, string limit, int value) => limit switch { nameof(PdfReaderLimits.MaxDecodedStreamLength) => limits with { MaxDecodedStreamLength = value }, + nameof(PdfReaderLimits.MaxStreamLength) => limits with { MaxStreamLength = value }, nameof(PdfReaderLimits.MaxObjectLength) => limits with { MaxObjectLength = value }, nameof(PdfReaderLimits.MaxXRefSectionLength) => limits with { MaxXRefSectionLength = value }, nameof(PdfReaderLimits.MaxXRefSectionCount) => limits with { MaxXRefSectionCount = value }, @@ -3165,6 +3326,7 @@ private static PdfReaderOptions TrailerCut() => private static int Bound(PdfReaderLimits limits, string limit) => limit switch { nameof(PdfReaderLimits.MaxDecodedStreamLength) => limits.MaxDecodedStreamLength, + nameof(PdfReaderLimits.MaxStreamLength) => limits.MaxStreamLength, nameof(PdfReaderLimits.MaxObjectLength) => limits.MaxObjectLength, nameof(PdfReaderLimits.MaxXRefSectionLength) => limits.MaxXRefSectionLength, nameof(PdfReaderLimits.MaxXRefSectionCount) => limits.MaxXRefSectionCount, @@ -3302,6 +3464,7 @@ private sealed record Case( public static Case For(string guard) => guard switch { "decoded stream" => DecodedStream(), + "stream length" => LongStream(), "object" => LongObject(), "section length" => LongSection(), "section count" => ManySections(), @@ -3338,6 +3501,29 @@ static int Decoded(PdfDocument document) => document.GetObject(new PdfObjectId(3)).AsStream().Required().Decode(document.Diagnostics).Length; } + private static Case LongStream() + { + // Ten thousand bytes of data no filter decodes, which a bound of a thousand cuts when they are read. + var file = new TestPdfBuilder() + .WithObject(1, Catalog) + .WithObject(2, Pages) + .Stream(3, string.Empty, new string('A', 10_000)) + .BuildClassic(rootNumber: 1); + + return new Case( + nameof(PdfReaderLimits.MaxStreamLength), + PdfDiagnosticCodes.LimitStreamLength, + file, + PdfReaderLimits.Default with { MaxStreamLength = 1000 }, + PdfReaderLimits.Default, + document => document.GetObject(new PdfObjectId(3)).AsStream().Required().GetRawBytes().Length.ToString(CultureInfo.InvariantCulture), + "10000", + "A stream's data runs past 1,000 bytes; only what lies within it was read.", + OffsetOf(file, "stream\n") + "stream\n".Length, + ThrowsOnOpening: false, + ReportsNothingElse: true); + } + private static Case LongFilterChain() { // Three layers of hex around a word, which a bound of two filters leaves one layer of. diff --git a/tests/corpus/manifest.schema.json b/tests/corpus/manifest.schema.json index c12668e0..aedc28d2 100644 --- a/tests/corpus/manifest.schema.json +++ b/tests/corpus/manifest.schema.json @@ -273,6 +273,7 @@ "filter.checksum-mismatch", "filter.unsupported", "limit.decoded-stream", + "limit.stream-length", "limit.object", "limit.xref-section-length", "limit.xref-section-count", @@ -440,6 +441,12 @@ "exclusiveMinimum": 268435456, "maximum": 2147483647 }, + "maxStreamLength": { + "type": "integer", + "description": "The most of one stream's encoded data read at once, in bytes.", + "exclusiveMinimum": 268435456, + "maximum": 2147483647 + }, "maxObjectLength": { "type": "integer", "description": "The longest object, the data of a stream aside, in bytes.", From 1b8782886629664c77ddfac4ea694a717a8dbd9a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 05:56:23 +0000 Subject: [PATCH 003/146] fix: end a stream's data where its object ends, found by one search wherever its window ends A stream's length was misjudged three ways. More than four bytes of white space before endstream, or a comment, made a right /Length a wrong one. A /Length the file cannot hold grew the window to MaxObjectLength even when an endstream was found: it read 21.5 MB for five bytes, reported limit.object and marked the object cut. And the search in the window took the first endstream it held, past the object's endobj and the next header: 49 bytes of object 6 for a stream of five. A declared length is confirmed by endstream after any white space and comments, the file read on as far as they run. Every other length - missing, unusable, past the file, or one endstream does not follow - goes to one search: the bytes the parser holds, then the file, up to the nearer of the first object header and the next object the index places, once for each stream. A stream's data never counts toward MaxObjectLength, and no window grows to find its end. With no endstream before that stop, the object's endobj plays its part: a declared length endobj follows is kept, and otherwise the endobj that ends the object before the stop ends the data, its end-of-line left out. Without one, a declared length that ends before the stop is kept, and the data otherwise runs to the stop, a declared length past it cut there. The parser alone applies the same rules to its buffer. A cross-reference stream whose /Length the chain could not read is searched as the window's search did, a carriage return kept and no header stopping it, now past the window rather than through one MaxTrailerLength bounds. Tests that declared a length past the next object, or cut stream data with MaxObjectLength, expect what the decisions give: the data ends at the object's endobj or at the next object, and stream data is cut by MaxStreamLength. PDFBox's pdfbox3947 object stream 417 keeps its declared 1,302 bytes, inside which its Flate data ends. Refs #174, #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../Documents/PdfReaderLimits.cs | 5 +- src/AdCodicem.Pdf/IO/HeaderText.cs | 231 +++++++ .../IO/IPdfStreamDataProvider.cs | 36 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 637 +++++++++--------- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 420 ++++++------ src/AdCodicem.Pdf/IO/StreamEnd.cs | 14 + src/AdCodicem.Pdf/IO/StreamEndKind.cs | 14 + src/AdCodicem.Pdf/IO/StreamEndSearch.cs | 27 +- src/AdCodicem.Pdf/IO/StreamEnds.cs | 189 ++++++ .../AdCodicem.Pdf.Tests/CorpusReadingTests.cs | 9 +- .../AdCodicem.Pdf.Tests/FilterDamageTests.cs | 95 ++- .../AdCodicem.Pdf.Tests/HostileInputTests.cs | 107 +++ tests/AdCodicem.Pdf.Tests/ParserTests.cs | 56 +- .../AdCodicem.Pdf.Tests/ReaderLimitsTests.cs | 11 +- .../AdCodicem.Pdf.Tests/StreamLengthTests.cs | 321 +++++++-- tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs | 14 +- 16 files changed, 1499 insertions(+), 687 deletions(-) create mode 100644 src/AdCodicem.Pdf/IO/HeaderText.cs create mode 100644 src/AdCodicem.Pdf/IO/StreamEnd.cs create mode 100644 src/AdCodicem.Pdf/IO/StreamEndKind.cs create mode 100644 src/AdCodicem.Pdf/IO/StreamEnds.cs diff --git a/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs b/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs index a8a8721c..204ac04c 100644 --- a/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs +++ b/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs @@ -88,7 +88,10 @@ public sealed record PdfReaderLimits /// directly or an object stream holds it. Defaults to 16 MB, which a direct array of a million references would need. /// /// - /// A value above is taken as that length. Reaching it reports + /// A stream's data never counts toward it, however its end is found: a /Length that is wrong, missing or past + /// the file has the data searched for its endstream past the bytes the object was parsed through, not read + /// through a larger window (#174); what the reader reads of the data at once is + /// 's. A value above is taken as that length. Reaching it reports /// , once for each object, and for each object an object /// stream holds, once for each definition of that stream. /// diff --git a/src/AdCodicem.Pdf/IO/HeaderText.cs b/src/AdCodicem.Pdf/IO/HeaderText.cs new file mode 100644 index 00000000..f3665ede --- /dev/null +++ b/src/AdCodicem.Pdf/IO/HeaderText.cs @@ -0,0 +1,231 @@ +using System.Globalization; +using AdCodicem.Pdf.Objects; + +namespace AdCodicem.Pdf.IO; + +/// +/// One read of a search for endstream as the rule for object headers sees it (): its +/// bytes, after a few that stand for what the search read before them. +/// +/// +/// Whether obj ends an object header depends on the runs of white space and of digits before it, which may run +/// back past the start of a read, as far as the start of the data. What the reads before hold is carried into each one +/// as a few bytes that decide every header as they would: the runs of white space or of digits that end where the read +/// starts, four at most — each run of white space as one space, each run of digits as the digits of its value, or of +/// 2,147,483,648 for any larger one, which no number nor generation reaches however many digits follow —, and the byte +/// before them. A header holds four such runs, and asks of the byte before them only whether it ends a token: no run +/// reads that byte. The start of the data is carried as a space, which ends a token as white space does and holds no +/// digit. The bytes carried have negative indexes, and each records where the run it stands for starts in the file. +/// None of the sizes below bounds what the file holds (ADR 34): a run of any length is carried, as what decides a +/// header — its kind and its value. +/// +internal ref struct HeaderText +{ + /// The most runs of white space or digits before obj that decide a header: those it holds. + private const int MaxRuns = 4; + + /// The most digits a run of digits is carried as: those of . + private const int MaxDigits = 10; + + /// The most bytes carried into a read: the byte before the runs, two runs of white space and two of digits. + internal const int CarriedLength = 1 + (2 * (1 + MaxDigits)); + + /// The value a run of digits is taken for when it is larger: past any object number and generation. + private const long Beyond = (long)PdfObjectId.MaxNumber + 1; + + private readonly Span _carried; + private readonly Span _carriedAt; + private int _carriedLength; + private ReadOnlySpan _bytes; + private long _at; + + /// Starts a search at , before which nothing is read. + /// Room for bytes carried into a read. + /// Room for where the run each of them stands for starts in the file. + /// Where the data starts in the file. + public HeaderText(Span carried, Span carriedAt, long dataStart) + { + _carried = carried; + _carriedAt = carriedAt; + _carried[0] = (byte)' '; + _carriedAt[0] = dataStart; + _carriedLength = 1; + } + + /// Gets the bytes of the read. + public readonly ReadOnlySpan Bytes => _bytes; + + /// Gets the index of the first byte carried into the read: the byte before the runs, which no run reads. + private readonly int FirstCarried => -_carriedLength; + + private readonly byte this[int index] => index >= 0 ? _bytes[index] : _carried[_carriedLength + index]; + + /// Takes , read at in the file, as the read that follows what is carried. + public void Read(ReadOnlySpan bytes, long at) + { + _bytes = bytes; + _at = at; + } + + /// Gives where the byte at is in the file, or, for a byte carried, where its run starts. + public readonly long PositionOf(int index) => index >= 0 ? _at + index : _carriedAt[_carriedLength + index]; + + /// + /// Carries into the next read, which starts at in this one, the runs of white space and + /// of digits that end there, and the byte before them. + /// + public void CarryTo(long position) + { + Span runAt = stackalloc long[MaxRuns]; + Span runValue = stackalloc long[MaxRuns]; + var end = (int)(position - _at); + var runs = 0; + + while (runs < MaxRuns && end > FirstCarried + 1 && + (PdfCharacters.IsDigit(this[end - 1]) || PdfCharacters.IsWhitespace(this[end - 1]))) + { + var digits = PdfCharacters.IsDigit(this[end - 1]); + var start = RunStart(end, digits); + runAt[runs] = PositionOf(start); + runValue[runs] = digits ? Value(start, end) : -1; + end = start; + runs++; + } + + // What is carried is read whole before it is written over. + var before = this[end - 1]; + _carriedAt[0] = PositionOf(end - 1); + _carried[0] = before; + var length = 1; + + for (var run = runs - 1; run >= 0; run--) + { + var written = 1; + + if (runValue[run] < 0) + { + _carried[length] = (byte)' '; + } + else + { + // The room carried has ten digits for each run of digits, and no value carried has more. + _ = runValue[run].TryFormat(_carried[length..], out written, default, CultureInfo.InvariantCulture); + } + + _carriedAt.Slice(length, written).Fill(runAt[run]); + length += written; + } + + _carriedLength = length; + } + + /// + /// Finds the first object header whose obj ends before in the read, and gives where + /// it starts in the file; -1 when there is none. + /// + /// + /// A header is N G obj at a token boundary, as the file's bytes hold it: white space, a delimiter or the start + /// of the data before the number; a run of digits whose value the parser takes for an object number, 1 to + /// 2,147,483,647, and one whose value it takes for a generation, 0 to 65,535, however many zeros lead them; white + /// space before each of the generation and obj, as much as the file holds; and white space, a delimiter or the + /// end of the search after obj. endobj is no header, nor 10 0 objx, nor digits a regular + /// character precedes, nor tokens a comment separates. The scan that rebuilds an index refuses endobj and + /// tokens a comment separates too, but takes 10 0 objx and digits a regular character precedes (#171). + /// A header lies whole in what the search reads, from the start of the data to the next object the index places: the + /// stream's own header, and whatever precedes the data, is never one, and one that the next object's offset cuts is + /// not seen, that offset stopping the search first. The rule reads nothing but the bytes, so where it stops cannot + /// depend on what was read before; and what a read's start cuts of a header is carried into it, and obj with + /// the byte after it is whole in the read that decides it, so where the reads start cannot change it either: a + /// candidate whose obj ends the read, where the search goes on, is left to the next read, which repeats it. + /// + /// The rule is no guard (ADR 34): only a stream whose endstream does not follow its declared length is + /// searched with it, which no valid file holds (ISO 32000-1, 7.3.8.1); a cross-reference stream whose /Length + /// the chain could not read, which a valid file may hold, is searched without it (#220). A valid stream's data may + /// hold text that reads as a header — an embedded PDF written uncompressed —: its length, confirmed, is never + /// searched. An invalid stream whose data holds such text before its endstream stops there, as one whose next + /// object came first, and its data ends there or before (#174); text that only looks like a header is read past, and + /// an endstream after it ends the data. + /// + /// + /// Where the endstream found in the read starts, or its length. + /// Whether the read ends where the search does, where obj may end. + public readonly long FindHeader(int before, bool endsRange) + { + var from = 0; + + while (true) + { + var index = _bytes[from..before].IndexOf("obj"u8); + + if (index < 0) + { + return -1; + } + + var keyword = from + index; + var after = keyword + 3; + + // What follows obj past the end of the read is the next read's to see, which repeats it: the end of the read + // ends the token only where the search ends. + if ((after < _bytes.Length ? !PdfCharacters.IsRegular(_bytes[after]) : endsRange) && + TryFindStart(keyword, out var start)) + { + return PositionOf(start); + } + + from = after; + } + } + + /// + /// Reads back from the obj at for the rest of an object header, as + /// takes one, and gives where its number starts. + /// + private readonly bool TryFindStart(int keyword, out int start) + { + var generationEnd = RunStart(keyword, digits: false); + var generationStart = RunStart(generationEnd, digits: true); + var numberEnd = RunStart(generationStart, digits: false); + start = RunStart(numberEnd, digits: true); + + // Each run holds a byte at least, what precedes the number ends a token, and the values are an object number's + // and a generation's. + return generationEnd < keyword && generationStart < generationEnd && numberEnd < generationStart && + start < numberEnd && !PdfCharacters.IsRegular(this[start - 1]) && + Value(start, numberEnd) is >= 1 and <= PdfObjectId.MaxNumber && + Value(generationStart, generationEnd) <= PdfObjectId.MaxGeneration; + } + + /// + /// Gives where the run of digits, or of white space, that ends at starts, the first byte + /// carried left out. + /// + private readonly int RunStart(int end, bool digits) + { + var start = end; + + while (start > FirstCarried + 1 && + (digits ? PdfCharacters.IsDigit(this[start - 1]) : PdfCharacters.IsWhitespace(this[start - 1]))) + { + start--; + } + + return start; + } + + /// + /// Gives the value of the digits from to , or when + /// it is larger. + /// + private readonly long Value(int start, int end) + { + var value = 0L; + + for (var index = start; index < end; index++) + { + value = Math.Min((value * 10) + (this[index] - '0'), Beyond); + } + + return value; + } +} diff --git a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs index ccc4e942..d3c200dd 100644 --- a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs +++ b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs @@ -20,36 +20,42 @@ internal interface IPdfStreamDataProvider PdfStreamData Create(long absoluteOffset, int length); /// - /// Determines whether an endstream keyword starts at , after at - /// most a few bytes of white space. The parser asks when that keyword may lie past the end of its buffer. + /// Determines whether an endstream keyword follows , after white space and + /// comments, however many (#174), and what follows it. The parser asks at the end of a stream's declared length when its + /// buffer cannot say: the data runs past it, or the white space after it does. /// - bool IsEndStreamAt(long absoluteOffset); - - /// - /// Determines whether an endstream keyword starts at , as - /// does, and what follows it. The parser asks at the end of a stream whose data runs - /// past its buffer, where it can see neither. - /// - /// Null when no endstream starts there; what follows it otherwise. + /// + /// The file is read on as far as the white space and comments run, the reads past the first counted with the searches + /// for endstream (); once those have read as much as they may, the + /// first read alone decides. + /// + /// Null when no endstream follows; what follows it otherwise. EndObjState? CheckEndStream(long absoluteOffset); /// - /// Looks for where the data of a stream that runs past the parser's buffer ends, when no endstream follows - /// its declared length: the first endstream after the start of the data, before the next object the file's - /// index places — as the file wrote it, or as the reader rebuilt it as the document opened when the file wrote none —, - /// before the first object header the file's bytes hold, and before the end of the file. + /// Looks for where the data of a stream ends when no endstream follows its declared length, or it has none: the + /// first endstream after the start of the data, in the bytes the parser holds and then in the file, before the + /// next object the file's index places — as the file wrote it, or as the reader rebuilt it as the document opened when + /// the file wrote none —, before the first object header the file's bytes hold, and before the end of the file. /// /// /// The file is searched once for each stream: asked again for the data at the same place — the stream parsed /// again, after the cache let it go or the index was rebuilt —, the provider answers what it found the first time. + /// When it finds no endstream, it says what the object's endobj does instead (#174): whether one follows + /// the declared length, where the one that ends the object before the stop lies, and whether the stretch holds one. /// /// /// The number of the object the stream is the value of, whose own entry in the index is no next object; 0 for a /// stream read alone. /// /// Where the data starts in the file. + /// Where the declared length ends in the file, or -1 when the stream has none it can take. /// The bytes of the data the parser holds, from its start, which are searched first. - StreamEndSearch FindEndStream(int number, long absoluteDataStart, ReadOnlySpan buffered); + /// + /// Whether the stream is a cross-reference stream whose /Length the chain could not read (#182): its rows may + /// spell an object header and end in a carriage return, so neither stops nor trims the search (#220). + /// + StreamEndSearch FindEndStream(int number, long absoluteDataStart, long declaredEnd, ReadOnlySpan buffered, bool deferred); /// /// Resolves the object a stream's /Length names, and says whether the file holds it and whether it could be diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 4c9def51..3feb8477 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -140,12 +140,18 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// /// How many times the length of the file the searches for endstream of one document may read, together, - /// before a stream whose declared length no endstream follows keeps it without a search. + /// before a stream whose declared length no endstream follows keeps it without a search past the bytes the + /// parser holds, and white space after a declared length is read no further than a first read. /// /// - /// Not a guard a valid file reaches (ADR 34): a valid file is never searched. Each search reads from a stream's - /// data to the first object header the bytes hold after it, or to the next object the index as written places, - /// whichever is nearer, and its reads grow from twice what the parser holds of the data + /// Not a guard a valid file reaches (ADR 34), argued here once for every read it counts. A valid file is read past the + /// parser's window in two ways only: a cross-reference stream whose /Length the chain cannot read (#220) is + /// searched once to its endstream, and white space after a declared length that runs past a first read is read + /// on once for each place (#174). The data of different sections lies apart, and so do the runs of white space after + /// different streams: together those reads cover the file once, and one read past each — about twice the file at + /// most. Every other search is of an invalid stream, whose endstream does not follow its declared length. Each + /// reads from a stream's data to the first object header the bytes hold after it, or to the next object the index as + /// written places, whichever is nearer, and its reads grow from twice what the parser holds of the data /// (): it reads at most about twice what its object spans, and the searches of /// objects whose headers follow one another read disjoint stretches — the file twice over at most. Only searches that /// share a stretch read more: objects that overlap, the header of one inside the dictionary of another, before its @@ -350,10 +356,17 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider private readonly Dictionary _streamLengthFaults = []; /// - /// What each search of the file for a stream's endstream found, by where the stream's data starts: the file - /// is searched once for each stream, however many times it is parsed. + /// What each search of the file for a stream's endstream found, by where the stream's data starts and whether + /// its /Length was one the chain could not read: the file is searched once for each stream, however many times + /// it is parsed. /// - private readonly Dictionary _endStreamSearches = []; + private readonly Dictionary<(long DataStart, bool Deferred), StreamEndSearch> _endStreamSearches = []; + + /// + /// What follows each place the reader read on past white space and comments to find an endstream (#174): read + /// once, however many times its stream is parsed. + /// + private readonly Dictionary _endStreamChecks = []; /// /// Where the data starts of each stream whose length fault a reading the reader kept reported: the stream is not @@ -1492,33 +1505,234 @@ PdfStreamData IPdfStreamDataProvider.Create(long absoluteOffset, int length) } /// - bool IPdfStreamDataProvider.IsEndStreamAt(long absoluteOffset) + EndObjState? IPdfStreamDataProvider.CheckEndStream(long absoluteOffset) { // A source is only asked for bytes it has, as GetWindow does: a subclass need not accept an offset // or a length that runs past its end. if (absoluteOffset < 0 || absoluteOffset >= _source.Length) { - return false; + return null; } - Span tail = stackalloc byte[PdfObjectParser.EndStreamLookahead]; - var available = (int)Math.Min(tail.Length, _source.Length - absoluteOffset); - var read = _source.Read(absoluteOffset, tail[..available]); - return PdfObjectParser.IsEndStreamAt(tail[..read], 0); + // One read decides for all but a stream whose white space runs past it: a few bytes, on the stack. + Span tail = stackalloc byte[PdfObjectParser.EndObjLookahead]; + var read = _source.Read(absoluteOffset, tail[..(int)Math.Min(tail.Length, _source.Length - absoluteOffset)]); + var end = StreamEnds.ReadEnd(tail[..read], absoluteOffset + read >= _source.Length); + + return end.Kind switch + { + StreamEndKind.None => null, + StreamEndKind.EndStream when end.EndObj != EndObjState.Unseen => end.EndObj, + _ => CheckEndStreamOn(absoluteOffset, end), + }; } - /// - EndObjState? IPdfStreamDataProvider.CheckEndStream(long absoluteOffset) + /// + /// Reads on, past what one read of holds, through the white space + /// and comments that may lead to an endstream at , or to what follows it: once + /// for each place, however often its stream is parsed. + /// + /// Where the data ends. + /// What the first read said: undecided, or an endstream after which it saw nothing. + private EndObjState? CheckEndStreamOn(long absoluteOffset, StreamEnd first) + { + if (_endStreamChecks.TryGetValue(absoluteOffset, out var known)) + { + return known; + } + + EndObjState? state; + var keywordEnd = absoluteOffset + first.KeywordEnd; + + if (first.Kind == StreamEndKind.Undecided) + { + var at = NextSignificant(absoluteOffset); + Span keyword = stackalloc byte[StreamEnds.EndStreamKeyword.Length + 1]; + var read = at < 0 ? 0 : _source.Read(at, keyword[..(int)Math.Min(keyword.Length, _source.Length - at)]); + + if (!keyword[..read].StartsWith(StreamEnds.EndStreamKeyword)) + { + // Once the searches have read as much as they may, a run the first read could not see past confirms nothing. + _endStreamChecks[absoluteOffset] = null; + return null; + } + + // "endstreamx" is no keyword, and is what follows the data. + if (read == keyword.Length && PdfCharacters.IsRegular(keyword[^1])) + { + _endStreamChecks[absoluteOffset] = EndObjState.Absent; + return EndObjState.Absent; + } + + keywordEnd = at + StreamEnds.EndStreamKeyword.Length; + } + + var next = NextSignificant(keywordEnd); + state = next < 0 ? EndObjState.Unseen : IsEndObjKeywordAt(next) ? EndObjState.Present : EndObjState.Absent; + _endStreamChecks[absoluteOffset] = state; + return state; + } + + /// + /// Determines whether an endobj keyword, whole, follows the declared length of a stream whose data starts at + /// , white space and comments between: in the bytes the parser holds when they say, in the + /// file otherwise. + /// + private bool EndObjFollows(long dataStart, long declaredEnd, ReadOnlySpan buffered) => + (declaredEnd - dataStart <= buffered.Length + ? StreamEnds.IsEndObjAt(buffered, (int)(declaredEnd - dataStart), reachesEndOfFile: dataStart + buffered.Length >= _source.Length) + : null) ?? IsEndObjAt(declaredEnd); + + /// + /// Gives where the data of a stream that has no endstream ends when its object's own endobj ends it before + /// : in the bytes the parser holds when they reach the stop, in the file otherwise. + /// + private long EndObjBefore(long dataStart, long stop, ReadOnlySpan buffered) + { + if (stop - dataStart > buffered.Length) + { + return FindEndObjBefore(dataStart, stop); + } + + var end = StreamEnds.EndObjBefore(buffered[..(int)(stop - dataStart)]); + return end < 0 ? -1 : dataStart + end; + } + + /// + /// Determines whether an endobj keyword, whole, follows after white space and + /// comments: the end of a stream's object that has no endstream. + /// + private bool IsEndObjAt(long absoluteOffset) { if (absoluteOffset < 0 || absoluteOffset >= _source.Length) { - return null; + return false; } Span tail = stackalloc byte[PdfObjectParser.EndObjLookahead]; - var available = (int)Math.Min(tail.Length, _source.Length - absoluteOffset); - var read = _source.Read(absoluteOffset, tail[..available]); - return PdfObjectParser.ReadStreamEnd(tail[..read], absoluteOffset + read >= _source.Length); + var read = _source.Read(absoluteOffset, tail[..(int)Math.Min(tail.Length, _source.Length - absoluteOffset)]); + + if (StreamEnds.IsEndObjAt(tail[..read], 0, absoluteOffset + read >= _source.Length) is { } decided) + { + return decided; + } + + var next = NextSignificant(absoluteOffset); + return next >= 0 && IsEndObjKeywordAt(next); + } + + /// + /// Gives where the data of a stream that has no endstream ends when its object's own endobj ends it: the + /// endobj, a whole keyword after , that only white space separates from + /// , the end-of-line before it left out; -1 when none does. + /// + /// Where the data starts in the file. + /// Where the search for the stream's endstream stopped: an object, or the end of the file. + private long FindEndObjBefore(long absoluteDataStart, long stop) + { + // The white space before the stop, read back a few bytes at a time and more as it runs on: the search read it + // already, and this reads it once more at most. + var end = stop; + var size = PdfObjectParser.EndObjLookahead; + + while (end > absoluteDataStart) + { + var from = Math.Max(absoluteDataStart, end - size); + using var window = _source.GetWindow(from, (int)(end - from)); + var span = window.Memory.Span; + _endStreamSearchBytes += span.Length; + + if (span.Length < end - from) + { + return -1; + } + + var last = span.LastIndexOfAnyExcept(PdfCharacters.Whitespace); + + if (last >= 0) + { + end = from + last + 1; + break; + } + + end = from; + size = (int)Math.Min(2L * size, EndStreamSearchChunk); + } + + // The keyword, the byte before it and the end-of-line before that. + var before = Math.Max(absoluteDataStart, end - StreamEnds.EndObjKeyword.Length - 3); + Span bytes = stackalloc byte[StreamEnds.EndObjKeyword.Length + 3]; + var length = (int)(end - before); + var held = _source.Read(before, bytes[..length]); + var dataEnd = held == length ? StreamEnds.EndObjEnding(bytes[..length], atDataStart: before == absoluteDataStart) : -1; + return dataEnd < 0 ? -1 : before + dataEnd; + } + + /// + /// Gives where the first byte at or after lies that is neither white space nor part of a + /// comment, reading the file on as far as they run, or the length of the file when they run to its end; -1 when the + /// searches for endstream have read as much as they may (), which these reads + /// count with, or the source gives less than it holds. + /// + private long NextSignificant(long position) + { + var size = PdfObjectParser.EndObjLookahead; + var inComment = false; + + while (position < _source.Length) + { + if (_endStreamSearchBytes >= EndStreamSearchPasses * _source.Length) + { + return -1; + } + + using var window = _source.GetWindow(position, (int)Math.Min(size, _source.Length - position)); + var span = window.Memory.Span; + _endStreamSearchBytes += span.Length; + + if (span.IsEmpty) + { + return -1; + } + + for (var i = 0; i < span.Length; i++) + { + var current = span[i]; + + if (inComment) + { + // A comment runs to the end of the line; both line endings occur in the wild. + inComment = current is not ((byte)'\r' or (byte)'\n'); + } + else if (current == (byte)'%') + { + inComment = true; + } + else if (!PdfCharacters.IsWhitespace(current)) + { + return position + i; + } + } + + position += span.Length; + size = (int)Math.Min(2L * size, EndStreamSearchChunk); + } + + return _source.Length; + } + + /// Determines whether endobj, a whole keyword, starts at . + private bool IsEndObjKeywordAt(long position) + { + if (position >= _source.Length) + { + return false; + } + + Span bytes = stackalloc byte[StreamEnds.EndObjKeyword.Length + 1]; + var read = _source.Read(position, bytes[..(int)Math.Min(bytes.Length, _source.Length - position)]); + return bytes[..read].StartsWith(StreamEnds.EndObjKeyword) && + (read == StreamEnds.EndObjKeyword.Length ? position + read >= _source.Length : !PdfCharacters.IsRegular(bytes[^1])); } /// @@ -1545,45 +1759,53 @@ bool IPdfStreamDataProvider.IsEndStreamAt(long absoluteOffset) /// /// /// This bound is no guard (ADR 34): a valid file's endstream follows its declared length (ISO 32000-1, - /// 7.3.8.1) — so do those of the 35,873 streams of the corpus whose length is confirmed, a gap of 0 to 2 bytes - /// between them —, and only an invalid file is searched at all. What it reads is the stretch between the stream and - /// the next object, once for each stream: the result is kept, and a stream parsed again — after the cache let it go, - /// or a rebuild of the index — is not searched again. What the searches read together is bounded by + /// 7.3.8.1) — so do those of the 35,716 streams of the corpus whose length is confirmed, a gap of 0 to 2 bytes + /// between them, counted over the final index of each of its 401 documents —, and only an invalid stream is searched, + /// but for a cross-reference stream whose /Length the chain could not read, which is searched with neither + /// stop the rule for headers makes nor a carriage return trimmed, as : its rows may spell + /// a header, and end in a carriage return. What it reads is the stretch between the stream and the next object, + /// once for each stream: the result is kept, and a stream parsed again — after the cache let it go, or a rebuild of + /// the index — is not searched again. What the searches read together is bounded by /// . /// /// - StreamEndSearch IPdfStreamDataProvider.FindEndStream(int number, long absoluteDataStart, ReadOnlySpan buffered) + StreamEndSearch IPdfStreamDataProvider.FindEndStream( + int number, long absoluteDataStart, long declaredEnd, ReadOnlySpan buffered, bool deferred) { - if (_endStreamSearches.TryGetValue(absoluteDataStart, out var known)) + if (_endStreamSearches.TryGetValue((absoluteDataStart, deferred), out var known)) { return known; } - StreamEndSearch search; + var nextObject = NextObjectStart(number, absoluteDataStart); - if (_endStreamSearchBytes >= EndStreamSearchPasses * _source.Length) - { - search = new StreamEndSearch(null, null, EndObjState.Unknown, Searched: false); - } - else - { - var nextObject = NextObjectStart(number, absoluteDataStart); - var dataEnd = SearchEndStream(absoluteDataStart, buffered, nextObject ?? _source.Length, out var header); + // Once the searches have read as much of the file as they may, the bytes the parser holds are searched alone: they + // cost no read. + var readsFile = _endStreamSearchBytes < EndStreamSearchPasses * _source.Length; + var dataEnd = SearchEndStream( + absoluteDataStart, buffered, nextObject ?? _source.Length, deferred, readsFile, out var header, out var searched, out var endObjSeen); - // A header the bytes hold is looked for only before the next object the index places: found, it is the nearer. - var stop = header >= 0 ? header : nextObject; + // A header the bytes hold is looked for only before the next object the index places: found, it is the nearer. + var stop = header >= 0 ? header : nextObject; + var end = stop ?? _source.Length; - // Past int.MaxValue bytes, no length the reader holds a stream by reaches the endstream found: only a file of - // more than 2 GB can put one there, and the declared length is kept, as when none is found. - search = dataEnd >= 0 && dataEnd - absoluteDataStart <= int.MaxValue - ? new StreamEndSearch( - (int)(dataEnd - absoluteDataStart), - stop, - ((IPdfStreamDataProvider)this).CheckEndStream(dataEnd) ?? EndObjState.Unseen) - : new StreamEndSearch(null, stop, EndObjState.Unknown); - } + // Past int.MaxValue bytes, no length the reader holds a stream by reaches the endstream found: only a file of more + // than 2 GB can put one there, and the stream is read as one whose endstream is not found. With none, what the + // object's endobj does is read once with the search, as long as the searches may read. + var search = dataEnd >= 0 && dataEnd - absoluteDataStart <= int.MaxValue + ? new StreamEndSearch( + (int)(dataEnd - absoluteDataStart), + stop, + ((IPdfStreamDataProvider)this).CheckEndStream(dataEnd) ?? EndObjState.Unseen) + : new StreamEndSearch(null, stop, EndObjState.Unknown, searched) + { + EndObjSeen = endObjSeen, + EndObjAfterDeclared = searched && declaredEnd >= absoluteDataStart && declaredEnd <= end && + EndObjFollows(absoluteDataStart, declaredEnd, buffered), + ObjectEnd = searched ? EndObjBefore(absoluteDataStart, end, buffered) : -1, + }; - _endStreamSearches[absoluteDataStart] = search; + _endStreamSearches[(absoluteDataStart, deferred)] = search; return search; } @@ -1682,15 +1904,33 @@ internal static int EndStreamSearchFirstRead(int buffered) => /// for what each read repeats, and each read is searched for both: a header only before the endstream it holds, /// with what carries into it of the reads before. /// - private long SearchEndStream(long dataStart, ReadOnlySpan buffered, long limit, out long header) + /// Where the data starts in the file. + /// The bytes of the data the parser holds, from its start. + /// Where the next object the index places starts, or the end of the file. + /// Whether the stream's /Length is one the chain could not read, which no header stops. + /// Whether the file past the bytes the parser holds may be read. + /// Where the header that stopped the search starts, or -1. + /// Whether the search reached where it stops, rather than the end of what it might read. + /// Whether the bytes searched, before a header that stopped the search, hold an endobj. + private long SearchEndStream( + long dataStart, + ReadOnlySpan buffered, + long limit, + bool deferred, + bool readsFile, + out long header, + out bool searched, + out bool endObjSeen) { Span carried = stackalloc byte[HeaderText.CarriedLength]; Span carriedAt = stackalloc long[HeaderText.CarriedLength]; var text = new HeaderText(carried, carriedAt, dataStart); + searched = true; var inBuffer = (int)Math.Clamp(limit - dataStart, 0, buffered.Length); text.Read(buffered[..inBuffer], dataStart); - var found = SearchStretch(text, endsRange: dataStart + inBuffer >= limit, out header); + var found = StreamEnds.SearchStretch(text, endsRange: dataStart + inBuffer >= limit, deferred, out header); + endObjSeen = StreamEnds.HoldsEndObj(buffered[..inBuffer], header >= 0 ? (int)(header - dataStart) : inBuffer); if (found >= 0 || header >= 0) { @@ -1704,6 +1944,12 @@ private long SearchEndStream(long dataStart, ReadOnlySpan buffered, long l return -1; } + if (!readsFile) + { + searched = false; + return -1; + } + // Each window starts before the end, and the next one further on than it: the loop ends at one of the returns. var position = dataStart + Math.Max(0, buffered.Length - EndStreamSearchOverlap); var size = EndStreamSearchFirstRead(buffered.Length); @@ -1721,7 +1967,8 @@ private long SearchEndStream(long dataStart, ReadOnlySpan buffered, long l } text.Read(span, position); - found = SearchStretch(text, endsRange: position + span.Length >= end, out header); + found = StreamEnds.SearchStretch(text, endsRange: position + span.Length >= end, deferred, out header); + endObjSeen |= StreamEnds.HoldsEndObj(span, header >= 0 ? (int)(header - position) : span.Length); if (found >= 0 || header >= 0 || position + span.Length >= end) { @@ -1734,269 +1981,6 @@ private long SearchEndStream(long dataStart, ReadOnlySpan buffered, long l } } - /// - /// Searches one read of a search for endstream — the parser's bytes, or a window of the file — for the first - /// endstream, and for the first object header before it: returns where the data ends in the file, the - /// end-of-line before the keyword left out, or -1 when the read holds no endstream or a header comes first, - /// whose start in the file is then , -1 otherwise. - /// - /// - /// The overlap between reads puts the end-of-line before a keyword first seen in a read inside it, unless the keyword - /// starts the data. - /// - /// The bytes read, after what the reads before carry into them. - /// Whether they end where the search does: at the next object the index places, or the end of the file. - /// Where the header that stopped the search starts in the file, or -1. - private static long SearchStretch(in HeaderText text, bool endsRange, out long header) - { - var span = text.Bytes; - var keyword = span.IndexOf("endstream"u8); - header = text.FindHeader(keyword >= 0 ? keyword : span.Length, endsRange); - - if (keyword < 0 || header >= 0) - { - return -1; - } - - if (keyword > 0 && span[keyword - 1] == (byte)'\n') - { - keyword--; - } - - if (keyword > 0 && span[keyword - 1] == (byte)'\r') - { - keyword--; - } - - return text.PositionOf(keyword); - } - - /// - /// One read of a search for endstream as the rule for object headers sees it (): its - /// bytes, after a few that stand for what the search read before them. - /// - /// - /// Whether obj ends an object header depends on the runs of white space and of digits before it, which may run - /// back past the start of a read, as far as the start of the data. What the reads before hold is carried into each one - /// as a few bytes that decide every header as they would: the runs of white space or of digits that end where the read - /// starts, four at most — each run of white space as one space, each run of digits as the digits of its value, or of - /// 2,147,483,648 for any larger one, which no number nor generation reaches however many digits follow —, and the byte - /// before them. A header holds four such runs, and asks of the byte before them only whether it ends a token: no run - /// reads that byte. The start of the data is carried as a space, which ends a token as white space does and holds no - /// digit. The bytes carried have negative indexes, and each records where the run it stands for starts in the file. - /// None of the sizes below bounds what the file holds (ADR 34): a run of any length is carried, as what decides a - /// header — its kind and its value. - /// - private ref struct HeaderText - { - /// The most runs of white space or digits before obj that decide a header: those it holds. - private const int MaxRuns = 4; - - /// The most digits a run of digits is carried as: those of . - private const int MaxDigits = 10; - - /// The most bytes carried into a read: the byte before the runs, two runs of white space and two of digits. - internal const int CarriedLength = 1 + (2 * (1 + MaxDigits)); - - /// The value a run of digits is taken for when it is larger: past any object number and generation. - private const long Beyond = (long)PdfObjectId.MaxNumber + 1; - - private readonly Span _carried; - private readonly Span _carriedAt; - private int _carriedLength; - private ReadOnlySpan _bytes; - private long _at; - - /// Starts a search at , before which nothing is read. - /// Room for bytes carried into a read. - /// Room for where the run each of them stands for starts in the file. - /// Where the data starts in the file. - public HeaderText(Span carried, Span carriedAt, long dataStart) - { - _carried = carried; - _carriedAt = carriedAt; - _carried[0] = (byte)' '; - _carriedAt[0] = dataStart; - _carriedLength = 1; - } - - /// Gets the bytes of the read. - public readonly ReadOnlySpan Bytes => _bytes; - - /// Gets the index of the first byte carried into the read: the byte before the runs, which no run reads. - private readonly int FirstCarried => -_carriedLength; - - private readonly byte this[int index] => index >= 0 ? _bytes[index] : _carried[_carriedLength + index]; - - /// Takes , read at in the file, as the read that follows what is carried. - public void Read(ReadOnlySpan bytes, long at) - { - _bytes = bytes; - _at = at; - } - - /// Gives where the byte at is in the file, or, for a byte carried, where its run starts. - public readonly long PositionOf(int index) => index >= 0 ? _at + index : _carriedAt[_carriedLength + index]; - - /// - /// Carries into the next read, which starts at in this one, the runs of white space and - /// of digits that end there, and the byte before them. - /// - public void CarryTo(long position) - { - Span runAt = stackalloc long[MaxRuns]; - Span runValue = stackalloc long[MaxRuns]; - var end = (int)(position - _at); - var runs = 0; - - while (runs < MaxRuns && end > FirstCarried + 1 && - (PdfCharacters.IsDigit(this[end - 1]) || PdfCharacters.IsWhitespace(this[end - 1]))) - { - var digits = PdfCharacters.IsDigit(this[end - 1]); - var start = RunStart(end, digits); - runAt[runs] = PositionOf(start); - runValue[runs] = digits ? Value(start, end) : -1; - end = start; - runs++; - } - - // What is carried is read whole before it is written over. - var before = this[end - 1]; - _carriedAt[0] = PositionOf(end - 1); - _carried[0] = before; - var length = 1; - - for (var run = runs - 1; run >= 0; run--) - { - var written = 1; - - if (runValue[run] < 0) - { - _carried[length] = (byte)' '; - } - else - { - // The room carried has ten digits for each run of digits, and no value carried has more. - _ = runValue[run].TryFormat(_carried[length..], out written, default, CultureInfo.InvariantCulture); - } - - _carriedAt.Slice(length, written).Fill(runAt[run]); - length += written; - } - - _carriedLength = length; - } - - /// - /// Finds the first object header whose obj ends before in the read, and gives where - /// it starts in the file; -1 when there is none. - /// - /// - /// A header is N G obj at a token boundary, as the file's bytes hold it: white space, a delimiter or the start - /// of the data before the number; a run of digits whose value the parser takes for an object number, 1 to - /// 2,147,483,647, and one whose value it takes for a generation, 0 to 65,535, however many zeros lead them; white - /// space before each of the generation and obj, as much as the file holds; and white space, a delimiter or the - /// end of the search after obj. endobj is no header, nor 10 0 objx, nor digits a regular - /// character precedes, nor tokens a comment separates. The scan that rebuilds an index refuses endobj and - /// tokens a comment separates too, but takes 10 0 objx and digits a regular character precedes (#171). - /// A header lies whole in what the search reads, from the start of the data to the next object the index places: the - /// stream's own header, and whatever precedes the data, is never one, and one that the next object's offset cuts is - /// not seen, that offset stopping the search first. The rule reads nothing but the bytes, so where it stops cannot - /// depend on what was read before; and what a read's start cuts of a header is carried into it, and obj with - /// the byte after it is whole in the read that decides it, so where the reads start cannot change it either: a - /// candidate whose obj ends the read, where the search goes on, is left to the next read, which repeats it. - /// - /// The rule is no guard (ADR 34): only a stream whose endstream does not follow its declared length is - /// searched, which no valid file holds (ISO 32000-1, 7.3.8.1). A valid stream's data may hold text that reads as a - /// header — an embedded PDF written uncompressed —: its length, confirmed, is never searched. An invalid stream whose - /// data holds such text before its endstream stops there, keeps its declared length and is reported, as one - /// whose next object came first; text that only looks like a header is read past, and an endstream after it - /// ends the data. - /// - /// - /// Where the endstream found in the read starts, or its length. - /// Whether the read ends where the search does, where obj may end. - public readonly long FindHeader(int before, bool endsRange) - { - var from = 0; - - while (true) - { - var index = _bytes[from..before].IndexOf(ObjKeyword); - - if (index < 0) - { - return -1; - } - - var keyword = from + index; - var after = keyword + ObjKeyword.Length; - - // What follows obj past the end of the read is the next read's to see, which repeats it: the end of the read - // ends the token only where the search ends. - if ((after < _bytes.Length ? !PdfCharacters.IsRegular(_bytes[after]) : endsRange) && - TryFindStart(keyword, out var start)) - { - return PositionOf(start); - } - - from = after; - } - } - - /// - /// Reads back from the obj at for the rest of an object header, as - /// takes one, and gives where its number starts. - /// - private readonly bool TryFindStart(int keyword, out int start) - { - var generationEnd = RunStart(keyword, digits: false); - var generationStart = RunStart(generationEnd, digits: true); - var numberEnd = RunStart(generationStart, digits: false); - start = RunStart(numberEnd, digits: true); - - // Each run holds a byte at least, what precedes the number ends a token, and the values are an object number's - // and a generation's. - return generationEnd < keyword && generationStart < generationEnd && numberEnd < generationStart && - start < numberEnd && !PdfCharacters.IsRegular(this[start - 1]) && - Value(start, numberEnd) is >= 1 and <= PdfObjectId.MaxNumber && - Value(generationStart, generationEnd) <= PdfObjectId.MaxGeneration; - } - - /// - /// Gives where the run of digits, or of white space, that ends at starts, the first byte - /// carried left out. - /// - private readonly int RunStart(int end, bool digits) - { - var start = end; - - while (start > FirstCarried + 1 && - (digits ? PdfCharacters.IsDigit(this[start - 1]) : PdfCharacters.IsWhitespace(this[start - 1]))) - { - start--; - } - - return start; - } - - /// - /// Gives the value of the digits from to , or when - /// it is larger. - /// - private readonly long Value(int start, int end) - { - var value = 0L; - - for (var index = start; index < end; index++) - { - value = Math.Min((value * 10) + (this[index] - '0'), Beyond); - } - - return value; - } - } - /// /// Closes the reader as its document is disposed: whatever asks it for an object or for a stream's data from now on /// is refused with an that names the document, and what it kept — the objects @@ -2156,7 +2140,7 @@ private void CheckDeferredLengths() { var stopped = EndStreamAfter(dataStart + deferred.Taken); - if (!((IPdfStreamDataProvider)this).IsEndStreamAt(dataStart + length)) + if (((IPdfStreamDataProvider)this).CheckEndStream(dataStart + length) is null) { message = string.Create(CultureInfo.InvariantCulture, $"The stream declared {length} bytes but ended after {deferred.Taken}."); } @@ -2176,17 +2160,15 @@ private void CheckDeferredLengths() } /// - /// Gives where the endstream keyword starts that the end-of-line or white space at - /// leads to, or itself when none follows within the gap the parser allows. + /// Gives where the endstream keyword starts that the white space and comments at + /// lead to, or itself when none follows them. /// private long EndStreamAfter(long position) { - Span tail = stackalloc byte[PdfObjectParser.EndStreamLookahead]; - var read = position < 0 || position >= _source.Length - ? 0 - : _source.Read(position, tail[..(int)Math.Min(tail.Length, _source.Length - position)]); - var keyword = tail[..read].IndexOf("endstream"u8); - return keyword < 0 ? position : position + keyword; + var at = position < 0 ? -1 : NextSignificant(position); + Span keyword = stackalloc byte[StreamEnds.EndStreamKeyword.Length]; + var read = at < 0 || at >= _source.Length ? 0 : _source.Read(at, keyword[..(int)Math.Min(keyword.Length, _source.Length - at)]); + return keyword[..read].SequenceEqual(StreamEnds.EndStreamKeyword) ? at : position; } /// @@ -3450,9 +3432,9 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect { previous = -1; - // The first window, of 64 KB, holds the data of all but the largest cross-reference streams, so the - // parser checks their /Length against the data rather than only at the end the /Length gives. The - // dictionary is the trailer, and grows its window no further than a trailer may. + // The first window, of 64 KB, holds the data of all but the largest cross-reference streams. The dictionary is the + // trailer, and grows its window no further than a trailer may: the data never counts toward it, and a /Length the + // chain cannot read has the data searched for its endstream past the window (#174, #220). var guards = _guardsReached; // A place the section may have been moved to is a guess: what is there and is no section leaves nothing of its @@ -4518,8 +4500,7 @@ private bool TryParseNumberedAt( /// Parses object at an exact offset, as does, but keeps nothing of the /// reading: what the parser notices is dropped, nothing is recorded, and an object that still runs past the window /// once allows no larger one is not read rather than reported, which - /// says. A stream is not read: its data is looked for in the window alone, and the method - /// returns false. + /// says. A stream is not read: whatever its data, the method returns false. /// private bool TryReadUnrecorded(PdfObjectId id, long offset, out PdfObject value, out bool pastGuard) { @@ -4546,8 +4527,8 @@ private bool TryReadUnrecorded(PdfObjectId id, long offset, out PdfObject value, // A header read whole that names another object, or another generation, is not this object, however far // its body runs, the chain reading each object under its reference's own number and generation: no guard - // hides it. - if (read && found != id) + // hides it. A stream is not read either, wherever its data ends. + if (read && (found != id || parsed is PdfStream)) { return false; } diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index efe31f00..22c9d91f 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -26,25 +26,18 @@ internal ref struct PdfObjectParser /// private const int MaxDepth = 128; - /// The most white space accepted between a stream's data and its endstream keyword. - private const int MaxEndStreamGap = 4; - - /// How many bytes after a stream's data can decide whether endstream follows it. - internal const int EndStreamLookahead = MaxEndStreamGap + 9; - /// - /// How many bytes after a stream's data can decide, besides whether endstream follows it, whether - /// endobj follows that: the white space and keyword allows, then the few - /// bytes of white space or comment files put between the two keywords, and endobj with the byte after it. + /// How many bytes after a stream's data the reader reads first to decide whether endstream follows it, and + /// whether endobj follows that: the few bytes of white space or comment files put before and between the two + /// keywords, and the keywords with the byte after each. /// /// - /// No guard (ADR 34): it bounds what the reader sees of an object's end, not what it reads. A file that puts more - /// between the keywords reads the same, and whether its object ends with endobj goes unseen and unjudged, as - /// past a window's edge. + /// No bound (ADR 34): a first read, not what the reader reads. White space and comments that run past it are read on as + /// far as they go (#174), and what follows them decides. /// internal const int EndObjLookahead = 64; - private static ReadOnlySpan EndStreamKeyword => "endstream"u8; + private static ReadOnlySpan EndStreamKeyword => StreamEnds.EndStreamKeyword; private readonly ReadOnlyMemory _memory; private readonly IPdfObjectSource? _source; @@ -63,27 +56,26 @@ internal ref struct PdfObjectParser /// Whether the parse met the end of its buffer, where it may have cut an object short. private bool _truncated; - /// - /// Whether a stream's /Length runs past what the file holds, which only a window reaching the end of the file - /// can settle: a larger buffer is asked for, though the parse met no edge, and what it reads after the stream is whole. - /// - private bool _lengthUnsettled; - /// /// Whether the end of the data was reported already: a cut leaves every construct around it open, and is reported /// once, for the innermost. /// private bool _cutReported; - private bool _endStreamMissing; + /// + /// Whether the last stream read kept a declared length that neither endstream nor endobj follows: where + /// its object ends is unknown, and its endobj is not judged. + /// + private bool _endObjUnknown; + private EndObjState _endObj; /// What was found of the last stream read, when the file did not confirm its length. private StreamLengthFault? _lengthFault; /// - /// What follows the endstream of the last stream read, when its data ran past the buffer and the provider - /// was asked; null otherwise. + /// What follows the data of the last stream read, when the provider was asked: what follows its endstream, or, + /// when it has none, whether its object's endobj ends it; null otherwise. /// private EndObjState? _endObjAfterStream; @@ -168,7 +160,7 @@ public static PdfObjectParser ForObjectStreamMember( /// buffer could read differently. A caller that can offer one should; a caller whose buffer is the /// whole input keeps what was read. /// - public readonly bool IsTruncated => _truncated || _lengthUnsettled; + public readonly bool IsTruncated => _truncated; /// /// Gets a value indicating whether the object stream member being read is written as a stream, or holds one: a @@ -282,7 +274,7 @@ public bool TryReadIndirectObject(out PdfObjectId id, out PdfObject value) // end, and its answer stands where the buffer has none (#55). _endObj = next.IsKeyword("endobj"u8) ? EndObjState.Present - : _endStreamMissing ? EndObjState.Unknown + : _endObjUnknown ? EndObjState.Unknown : !unseen ? EndObjState.Absent : value is PdfStream && _endObjAfterStream is { } afterStream ? afterStream : EndObjState.Unseen; @@ -611,6 +603,19 @@ private PdfObject ParseDictionaryOrStream(int depth, int openedAt) return dictionary; } + /// + /// Reads a stream's data, whose stream keyword the lexer just read, and finds where it ends (#55, #174). + /// + /// + /// The declared length is confirmed by an endstream after it, white space and comments between, however many. + /// Unconfirmed, or with no length to confirm, the data is searched for its first endstream, which ends it: in the + /// bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold and the next + /// object the index places, or to the end of the data. Stream data never counts toward the bytes the parser is given + /// to read an object through: a window is never grown to find an endstream. With none before the stop, the + /// object's endobj plays its part: it ends the data where it follows the declared length, or where it ends the + /// object before the stop; without it, a declared length that ends before the stop is kept, and the data otherwise + /// runs to the stop. + /// private PdfStream ReadStream(PdfDictionary dictionary) { var span = _memory.Span; @@ -621,6 +626,7 @@ private PdfStream ReadStream(PdfDictionary dictionary) var dataStart = _lexer.Position; _lengthFault = null; _endObjAfterStream = null; + _endObjUnknown = false; // The end-of-line after "stream" is not data. A buffer that ends on the keyword, or on the carriage // return of a CR LF, cannot say where the data starts; a larger one can. @@ -632,129 +638,227 @@ private PdfStream ReadStream(PdfDictionary dictionary) var declared = ReadLength(dictionary); var length = declared.Form == StreamLengthForm.Integer ? (int)declared.Value.GetValueOrDefault() : -1; - var beyondBuffer = length >= 0 && dataStart + (long)length > span.Length; - - if (beyondBuffer && _streamData is not null) + if (_truncated) { - // The data lives past the window the reader gave us, into what the file holds: the file is asked - // whether endstream follows the declared length (#55). An attempt the reader makes again, its buffer - // too short to say where the data starts, asks nothing. - if (_baseOffset + dataStart + (long)length <= _streamData.SourceLength) - { - return IsTruncated - ? Finish(dictionary, dataStart, length, span.Length) - : ReadPastBuffer(_streamData, dictionary, declared, dataStart, length); - } - - // Unless the file cannot hold it. Then only a window that reaches the end of the file can say - // whether the data stops at an "endstream" — the length is wrong — or at the end of the file — - // the stream is cut —, and the search below says which once the reader has offered one. The parse - // met no edge for that: what it reads after an endstream the search finds is whole. - _lengthUnsettled = true; + return Finish(dictionary, dataStart, Math.Max(length, 0), span.Length); } - if (length < 0 || beyondBuffer || !ConfirmsLength(span, dataStart + length)) - { - // A length read once the cross-reference chain is keeps a carriage return before the line feed as data, which it - // may be: rows can end in 0x0D, and the length, once read, is checked against what was taken (#182). - var recovered = FindEndStream(span, dataStart, keepCarriageReturn: declared.Form == StreamLengthForm.Deferred); + var confirmed = ConfirmsLength(span, dataStart, length); - if (recovered < 0) - { - _truncated = true; + if (confirmed is null) + { + // Undecided, the buffer a window the parser alone was given: a larger one says. + _truncated = true; + return Finish(dictionary, dataStart, length, span.Length); + } - // An endobj after the data says the object ends there, without its endstream, and where it ends is - // then the reader's guess; with none, the file ends inside the object, and so without its endobj. - _endStreamMissing = span[dataStart..].IndexOf("endobj"u8) >= 0; - var rest = span.Length - dataStart; + if (confirmed == true) + { + return Finish(dictionary, dataStart, length, span.Length); + } - ReportLengthFault( - PdfDiagnosticCodes.StreamTruncated, - _endStreamMissing - ? string.Create( - CultureInfo.InvariantCulture, - $"The stream has no endstream before the endobj that follows its data; the {rest} bytes to the end of the file are taken as its data.") - : "A stream ran past the end of the file.", - Fault(declared, dataStart, rest, null, _endStreamMissing ? EndStreamState.MissingBeforeEndObj : EndStreamState.MissingBeforeEndOfFile, null)); + // A cross-reference stream whose /Length the chain could not read keeps a carriage return before the line feed as + // data, which it may be — rows can end in 0x0D, and the length, once read, is checked against what was taken —, + // and is not stopped by rows that spell an object header (#182, #220). + var deferred = declared.Form == StreamLengthForm.Deferred; + var declaredEnd = length < 0 ? -1 : _baseOffset + dataStart + length; + var search = _streamData is not null + ? _streamData.FindEndStream(_objectNumber, _baseOffset + dataStart, declaredEnd, span[dataStart..], deferred) + : SearchBuffer(span, dataStart, declaredEnd, deferred); - return Finish(dictionary, dataStart, rest, span.Length); - } + if (search is not { } settled) + { + _truncated = true; + return Finish(dictionary, dataStart, Math.Max(length, 0), span.Length); + } - if (length != recovered) + if (settled.Length is { } found) + { + if (length != found) { ReportLengthFault( PdfDiagnosticCodes.StreamLengthInvalid, - DescribeRecovered(declared, recovered), - Fault(declared, dataStart, recovered, recovered, EndStreamState.Found, null)); + DescribeRecovered(declared, found), + Fault(declared, dataStart, found, found, EndStreamState.Found, null)); } - length = recovered; + _endObjAfterStream = _streamData is null ? null : settled.EndObj; + return Finish(dictionary, dataStart, found, span.Length); } - return Finish(dictionary, dataStart, length, span.Length); + return FinishWithoutEndStream(dictionary, declared, dataStart, length, settled); } /// - /// Reads a stream whose declared length runs past the buffer, into what the file holds (#55). The file is asked - /// whether endstream follows the declared length, and when it does not, searched for the first one after the - /// start of the data, before the next object or the end of the file: found, it ends the data; not found, or not - /// searched for once the document's searches have read as much as they may, the declared length is kept. Either is - /// reported, and the file answers what follows the endstream too. + /// Ends the data of a stream whose search found no endstream before it stopped (#174): at the declared length, + /// when its object's endobj follows it; at that endobj, when it ends the object before the stop; at the + /// declared length, when it ends before the stop; at the stop otherwise — the next object, or the end of the data. /// - private PdfStream ReadPastBuffer( - IPdfStreamDataProvider provider, PdfDictionary dictionary, DeclaredLength declared, int dataStart, int length) + private PdfStream FinishWithoutEndStream( + PdfDictionary dictionary, DeclaredLength declared, int dataStart, int length, StreamEndSearch search) { var span = _memory.Span; + var endOfData = _streamData is null ? span.Length : _streamData.SourceLength - _baseOffset; + var stop = search.NextObject is { } next ? next - _baseOffset : endOfData; + var fits = length >= 0 && dataStart + (long)length <= stop; + var upToStop = (int)Math.Min(stop - dataStart, int.MaxValue); + var where = search.NextObject is { } nextObject + ? string.Create(CultureInfo.InvariantCulture, $"before the next object, at {nextObject}") + : "before the end of the file"; + var missing = search.NextObject is null ? EndStreamState.MissingBeforeEndOfFile : EndStreamState.MissingBeforeNextObject; - if (provider.CheckEndStream(_baseOffset + dataStart + (long)length) is { } afterEndStream) + if (!search.Searched) { - _endObjAfterStream = afterEndStream; - return Finish(dictionary, dataStart, length, span.Length); + // The document's searches read as much of the file as they may: what lies past the bytes held is not looked at. + var kept = fits ? length : upToStop; + ReportLengthFault( + PdfDiagnosticCodes.StreamLengthInvalid, + fits + ? string.Create( + CultureInfo.InvariantCulture, + $"The stream declared {length} bytes, and no endstream follows them; the declared length is kept without a search, the document's searches for endstream having read as much of the file as they may.") + : string.Create( + CultureInfo.InvariantCulture, + $"{Describe(declared)}, and no endstream follows its data in the bytes read with its object; the {kept} bytes {(search.NextObject is null ? "to the end of the file" : "up to the next object")} are taken as its data without a search, the document's searches for endstream having read as much of the file as they may."), + Fault(declared, dataStart, kept, null, EndStreamState.NotSearched, null)); + + _endObjUnknown = true; + return Finish(dictionary, dataStart, kept, span.Length); } - var search = provider.FindEndStream(_objectNumber, _baseOffset + dataStart, span[dataStart..]); - _endObjAfterStream = search.EndObj; + if (fits && search.EndObjAfterDeclared) + { + ReportLengthFault( + PdfDiagnosticCodes.StreamTruncated, + string.Create( + CultureInfo.InvariantCulture, + $"The stream has no endstream: its object's endobj follows the {length} bytes it declared, which are taken as its data."), + Fault(declared, dataStart, length, null, EndStreamState.MissingBeforeEndObj, null)); + + _endObjAfterStream = EndObjState.Present; + return Finish(dictionary, dataStart, length, span.Length); + } - if (search.Length is { } found) + if (search.ObjectEnd >= 0) { + var taken = (int)Math.Min(search.ObjectEnd - _baseOffset - dataStart, int.MaxValue); ReportLengthFault( - PdfDiagnosticCodes.StreamLengthInvalid, - string.Create(CultureInfo.InvariantCulture, $"The stream declared {length} bytes but ended after {found}."), - Fault(declared, dataStart, found, found, EndStreamState.Found, null)); + PdfDiagnosticCodes.StreamTruncated, + string.Create( + CultureInfo.InvariantCulture, + $"{Describe(declared)}, and no endstream follows its data {where}: its object's endobj ends it, after {taken} bytes."), + Fault(declared, dataStart, taken, null, EndStreamState.MissingBeforeEndObj, search.NextObject)); - return Finish(dictionary, dataStart, found, span.Length); + _endObjAfterStream = EndObjState.Present; + return Finish(dictionary, dataStart, taken, span.Length); } - if (!search.Searched) + if (fits) { ReportLengthFault( PdfDiagnosticCodes.StreamLengthInvalid, string.Create( CultureInfo.InvariantCulture, - $"The stream declared {length} bytes, and no endstream follows them; the declared length is kept without a search, the document's searches for endstream having read as much of the file as they may."), - Fault(declared, dataStart, length, null, EndStreamState.NotSearched, null)); + $"The stream declared {length} bytes, and no endstream follows them {where}; the declared length is kept."), + Fault(declared, dataStart, length, null, missing, search.NextObject)); + // Where the object ends is unknown, and its endobj is not judged. + _endObjUnknown = true; return Finish(dictionary, dataStart, length, span.Length); } - var before = search.NextObject is { } next - ? string.Create(CultureInfo.InvariantCulture, $"before the next object, at {next}") - : "before the end of the file"; - + // The data runs to the next object, which no declared length passes, or to the end of the file. The object lacks its + // endobj, unless what is taken as its data holds one, which may be the object's own: where it ends is then unknown. ReportLengthFault( - PdfDiagnosticCodes.StreamLengthInvalid, + search.NextObject is null ? PdfDiagnosticCodes.StreamTruncated : PdfDiagnosticCodes.StreamLengthInvalid, string.Create( CultureInfo.InvariantCulture, - $"The stream declared {length} bytes, and no endstream follows them {before}; the declared length is kept."), - Fault( - declared, - dataStart, - length, - null, - search.NextObject is null ? EndStreamState.MissingBeforeEndOfFile : EndStreamState.MissingBeforeNextObject, - search.NextObject)); - - return Finish(dictionary, dataStart, length, span.Length); + $"{Describe(declared)}, and no endstream follows its data {where}; the {upToStop} bytes {(search.NextObject is null ? "to the end of the file" : "up to that object")} are taken as its data."), + Fault(declared, dataStart, upToStop, null, missing, search.NextObject)); + + _endObjUnknown = search.EndObjSeen; + _endObjAfterStream = search.EndObjSeen ? EndObjState.Unknown : EndObjState.Absent; + return Finish(dictionary, dataStart, upToStop, span.Length); + } + + /// + /// Determines whether the declared length is confirmed by an endstream after it, white space and comments + /// between, however many: in the buffer, or, where the buffer cannot say, in the file, which the provider reads on as + /// far as they run (#174). Null when neither can say: the buffer is a window given to the parser alone. + /// + private bool? ConfirmsLength(ReadOnlySpan span, int dataStart, int length) + { + if (length < 0) + { + return false; + } + + var dataEnd = dataStart + (long)length; + + if (dataEnd <= span.Length) + { + var end = StreamEnds.ReadEnd(span[(int)dataEnd..], reachesEndOfFile: _endsData); + + if (end.Kind != StreamEndKind.Undecided) + { + return end.Kind == StreamEndKind.EndStream; + } + } + else if (_endsData) + { + return false; + } + + if (_streamData is null) + { + return null; + } + + if (_baseOffset + dataEnd > _streamData.SourceLength) + { + return false; + } + + _endObjAfterStream = _streamData.CheckEndStream(_baseOffset + dataEnd); + return _endObjAfterStream is not null; + } + + /// + /// Searches the buffer of a parser the reader does not back for the first endstream after the start of the data, + /// up to the first object header it holds, as the reader searches the file, and says what the object's endobj + /// does when it finds none; null when the buffer holds neither, and is a window a larger one may show more of. + /// + private readonly StreamEndSearch? SearchBuffer(ReadOnlySpan span, int dataStart, long declaredEnd, bool deferred) + { + Span carried = stackalloc byte[HeaderText.CarriedLength]; + Span carriedAt = stackalloc long[HeaderText.CarriedLength]; + var start = _baseOffset + dataStart; + var text = new HeaderText(carried, carriedAt, start); + text.Read(span[dataStart..], start); + + var dataEnd = StreamEnds.SearchStretch(text, endsRange: _endsData, deferred, out var header); + + if (dataEnd >= 0) + { + return new StreamEndSearch((int)(dataEnd - start), null, EndObjState.Unseen); + } + + if (header < 0 && !_endsData) + { + return null; + } + + var stop = header < 0 ? span.Length : (int)(header - _baseOffset); + var stretch = span[dataStart..stop]; + var objectEnd = StreamEnds.EndObjBefore(stretch); + + return new StreamEndSearch(null, header < 0 ? null : header, EndObjState.Unknown) + { + EndObjSeen = StreamEnds.HoldsEndObj(stretch, stretch.Length), + EndObjAfterDeclared = declaredEnd >= start && declaredEnd - _baseOffset <= stop && + StreamEnds.IsEndObjAt(span[..stop], (int)(declaredEnd - _baseOffset), reachesEndOfFile: true) == true, + ObjectEnd = objectEnd < 0 ? -1 : start + objectEnd, + }; } /// @@ -835,16 +939,23 @@ private readonly DeclaredLength ReadLength(PdfDictionary dictionary) }; /// Says what was wrong with a stream's /Length, its endstream found after bytes. - private static string DescribeRecovered(DeclaredLength declared, int recovered) + private static string DescribeRecovered(DeclaredLength declared, int recovered) => declared.Form == StreamLengthForm.Integer + ? string.Create(CultureInfo.InvariantCulture, $"The stream declared {declared.Value} bytes but ended after {recovered}.") + : string.Create(CultureInfo.InvariantCulture, $"{Describe(declared)}; its data ends after {recovered} bytes."); + + /// + /// Says how a stream's /Length was written, as a report starts: the length it declared, or why it gives none. + /// + private static string Describe(DeclaredLength declared) { if (declared.Form == StreamLengthForm.Integer) { - return string.Create(CultureInfo.InvariantCulture, $"The stream declared {declared.Value} bytes but ended after {recovered}."); + return string.Create(CultureInfo.InvariantCulture, $"The stream declared {declared.Value} bytes"); } if (declared.Form == StreamLengthForm.Absent) { - return string.Create(CultureInfo.InvariantCulture, $"The stream has no /Length; its data ends after {recovered} bytes."); + return "The stream has no /Length"; } var written = declared.Reference is not { } reference @@ -856,7 +967,7 @@ private static string DescribeRecovered(DeclaredLength declared, int recovered) _ => Names(reference, "which holds " + Value(declared)), }; - return string.Create(CultureInfo.InvariantCulture, $"The stream's /Length {written}; its data ends after {recovered} bytes."); + return "The stream's /Length " + written; static string Names(PdfObjectId reference, string what) => string.Create(CultureInfo.InvariantCulture, $"names object {reference.Number} {reference.Generation}, {what}"); @@ -934,99 +1045,6 @@ private PdfStream Finish(PdfDictionary dictionary, int dataStart, int length, in return new PdfStream(dictionary, data); } - /// - /// Determines whether the declared length is confirmed by an endstream keyword where the data - /// ends. When the keyword may lie across the end of the buffer, the file is asked rather than the buffer: - /// a stream whose data ends a few bytes short of the window's edge is not a truncated stream. - /// - private readonly bool ConfirmsLength(ReadOnlySpan span, int dataEnd) - { - if (IsEndStreamAt(span, dataEnd)) - { - return true; - } - - return _streamData is not null && - dataEnd + (long)EndStreamLookahead > span.Length && - _streamData.IsEndStreamAt(_baseOffset + dataEnd); - } - - /// - /// Determines whether endstream starts at , after at most - /// bytes of white space. - /// - internal static bool IsEndStreamAt(ReadOnlySpan span, long position) - { - if (position < 0 || position > span.Length) - { - return false; - } - - var index = (int)position; - - // Conforming files put an end-of-line before "endstream"; some put nothing, some put spaces. - var limit = Math.Min(index + MaxEndStreamGap, span.Length); - while (index < limit && PdfCharacters.IsWhitespace(span[index])) - { - index++; - } - - return span[index..].StartsWith(EndStreamKeyword); - } - - /// - /// Determines whether endstream starts at the start of , after at most - /// bytes of white space, and what follows the keyword: endobj, another token, - /// or nothing the bytes show — unless they reach the end of the file, which then follows it. - /// - /// Null when no endstream starts there. - internal static EndObjState? ReadStreamEnd(ReadOnlySpan bytes, bool reachesEndOfFile) - { - if (!IsEndStreamAt(bytes, 0)) - { - return null; - } - - // Read as the parser reads what follows a stream: "endstreamx" is no keyword, and is what follows the data. - var lexer = new PdfLexer(bytes); - var keyword = lexer.Read(); - var next = keyword.IsKeyword(EndStreamKeyword) ? lexer.Read() : keyword; - - if (!reachesEndOfFile && (next.Kind == PdfTokenKind.EndOfInput || next.End >= bytes.Length)) - { - return EndObjState.Unseen; - } - - return next.IsKeyword("endobj"u8) ? EndObjState.Present : EndObjState.Absent; - } - - /// - /// Finds where a stream really ends when its declared length is wrong, and returns its true length. - /// - private static int FindEndStream(ReadOnlySpan span, int dataStart, bool keepCarriageReturn = false) - { - var index = span[dataStart..].IndexOf(EndStreamKeyword); - if (index < 0) - { - return -1; - } - - var end = dataStart + index; - - // The end-of-line that precedes the keyword belongs to the syntax, not to the data. - if (end > dataStart && span[end - 1] == (byte)'\n') - { - end--; - } - - if (!keepCarriageReturn && end > dataStart && span[end - 1] == (byte)'\r') - { - end--; - } - - return end - dataStart; - } - /// /// Consumes a container whose contents are being discarded, keeping nesting balanced: the one that opens at /// , inside containers. diff --git a/src/AdCodicem.Pdf/IO/StreamEnd.cs b/src/AdCodicem.Pdf/IO/StreamEnd.cs new file mode 100644 index 00000000..b1c9cfb2 --- /dev/null +++ b/src/AdCodicem.Pdf/IO/StreamEnd.cs @@ -0,0 +1,14 @@ +namespace AdCodicem.Pdf.IO; + +/// What the bytes after a stream's data say of its end (). +/// Whether endstream follows the data, does not, or the bytes cannot say. +/// What follows the endstream, when one does. +/// Where the endstream token ends in the bytes, when one does. +internal readonly record struct StreamEnd(StreamEndKind Kind, EndObjState EndObj = EndObjState.Unseen, int KeywordEnd = 0) +{ + /// No endstream follows the data. + public static StreamEnd None => new(StreamEndKind.None); + + /// The bytes end in the white space, the comment or the keyword that may lead to endstream. + public static StreamEnd Undecided => new(StreamEndKind.Undecided); +} diff --git a/src/AdCodicem.Pdf/IO/StreamEndKind.cs b/src/AdCodicem.Pdf/IO/StreamEndKind.cs new file mode 100644 index 00000000..e02e7fc9 --- /dev/null +++ b/src/AdCodicem.Pdf/IO/StreamEndKind.cs @@ -0,0 +1,14 @@ +namespace AdCodicem.Pdf.IO; + +/// Whether the bytes after a stream's data hold its endstream. +internal enum StreamEndKind : byte +{ + /// Something else follows the data, after white space and comments. + None, + + /// endstream follows the data, after white space and comments. + EndStream, + + /// The bytes end before they can say: in white space, a comment, or what may be the keyword's start. + Undecided, +} diff --git a/src/AdCodicem.Pdf/IO/StreamEndSearch.cs b/src/AdCodicem.Pdf/IO/StreamEndSearch.cs index bfcb2f65..48f28739 100644 --- a/src/AdCodicem.Pdf/IO/StreamEndSearch.cs +++ b/src/AdCodicem.Pdf/IO/StreamEndSearch.cs @@ -15,7 +15,28 @@ namespace AdCodicem.Pdf.IO; /// is unknown. /// /// -/// Whether the file was searched at all: it is not once the document's searches have read as much of it as they may, -/// and then nothing was found. +/// Whether the file was searched as far as the search stops: past the bytes the parser holds, it is not once the +/// document's searches have read as much of it as they may, and then nothing past them was found. /// -internal readonly record struct StreamEndSearch(int? Length, long? NextObject, EndObjState EndObj, bool Searched = true); +internal readonly record struct StreamEndSearch(int? Length, long? NextObject, EndObjState EndObj, bool Searched = true) +{ + /// + /// Gets where the data ends, in the file, when no endstream was found and the object's own endobj, which + /// only white space separates from where the search stopped, ends it, the end-of-line before it left out; -1 when none + /// does. + /// + public long ObjectEnd { get; init; } = -1; + + /// + /// Gets a value indicating whether, no endstream found, an endobj follows the declared length, white + /// space and comments between, before where the search stopped. + /// + public bool EndObjAfterDeclared { get; init; } + + /// + /// Gets a value indicating whether, no endstream found, the bytes searched hold an endobj keyword + /// anywhere: the object may then end inside what is taken as its data, and whether it ends with endobj is + /// unknown. + /// + public bool EndObjSeen { get; init; } +} diff --git a/src/AdCodicem.Pdf/IO/StreamEnds.cs b/src/AdCodicem.Pdf/IO/StreamEnds.cs new file mode 100644 index 00000000..81541235 --- /dev/null +++ b/src/AdCodicem.Pdf/IO/StreamEnds.cs @@ -0,0 +1,189 @@ +namespace AdCodicem.Pdf.IO; + +/// +/// The rules that say where a stream's data ends, read over bytes: whether endstream follows the data, where the +/// first one after its start lies, and whether an endobj ends the object. The parser applies them to the bytes it +/// holds, and the reader to the file past them (), so that a stream reads the same +/// wherever its window ends (#174). +/// +internal static class StreamEnds +{ + /// The keyword that ends a stream's data. + public static ReadOnlySpan EndStreamKeyword => "endstream"u8; + + /// The keyword that ends an object. + public static ReadOnlySpan EndObjKeyword => "endobj"u8; + + /// + /// Gives where the first byte at or after lies that is neither white space nor part of a + /// comment, or the length of when they run to its end. ISO 32000-1 (7.2.3) makes a comment + /// outside a string or a stream's data count as white space. + /// + public static int SkipBlanks(ReadOnlySpan bytes, int index) + { + var lexer = new PdfLexer(bytes, index); + lexer.SkipWhitespaceAndComments(); + return lexer.Position; + } + + /// + /// Reads, at the start of , whether endstream follows a stream's data — after white + /// space and comments, as many as there are (#174) — and what follows the keyword: endobj, another token, or + /// nothing the bytes show. + /// + /// The bytes after the data. + /// Whether the bytes end where the file does, which then follows what they hold. + public static StreamEnd ReadEnd(ReadOnlySpan bytes, bool reachesEndOfFile) + { + var lexer = new PdfLexer(bytes); + lexer.SkipWhitespaceAndComments(); + var at = lexer.Position; + var rest = bytes[at..]; + + if (!rest.StartsWith(EndStreamKeyword)) + { + // White space, a comment or the start of the keyword that runs to the end of the bytes may lead to it past them. + return !reachesEndOfFile && EndStreamKeyword.StartsWith(rest) ? StreamEnd.Undecided : StreamEnd.None; + } + + // Read as the parser reads what follows a stream: "endstreamx" is no keyword, and is what follows the data. + var keyword = lexer.Read(); + + if (keyword.End >= bytes.Length && !reachesEndOfFile) + { + return StreamEnd.Undecided; + } + + if (!keyword.IsKeyword(EndStreamKeyword)) + { + return new StreamEnd(StreamEndKind.EndStream, EndObjState.Absent, keyword.End); + } + + var next = lexer.Read(); + + if (!reachesEndOfFile && (next.Kind == PdfTokenKind.EndOfInput || next.End >= bytes.Length)) + { + return new StreamEnd(StreamEndKind.EndStream, EndObjState.Unseen, keyword.End); + } + + return new StreamEnd( + StreamEndKind.EndStream, next.IsKeyword(EndObjKeyword) ? EndObjState.Present : EndObjState.Absent, keyword.End); + } + + /// + /// Determines whether endobj, a whole keyword, starts at once white space and comments are + /// skipped; null when they, or the keyword, run to the end of , which do not end the file. + /// + public static bool? IsEndObjAt(ReadOnlySpan bytes, int index, bool reachesEndOfFile) + { + var at = SkipBlanks(bytes, index); + var rest = bytes[at..]; + + if (rest.Length <= EndObjKeyword.Length) + { + if (!reachesEndOfFile && EndObjKeyword.StartsWith(rest)) + { + return null; + } + + return rest.SequenceEqual(EndObjKeyword); + } + + return rest.StartsWith(EndObjKeyword) && !PdfCharacters.IsRegular(rest[EndObjKeyword.Length]); + } + + /// + /// Determines whether the first bytes of hold an endobj + /// keyword: a read of a search that may end inside its object. + /// + public static bool HoldsEndObj(ReadOnlySpan bytes, int before) => + bytes[..Math.Clamp(before, 0, bytes.Length)].IndexOf(EndObjKeyword) >= 0; + + /// + /// Gives where the data of a stream that has no endstream ends when its object's own endobj ends it: the + /// endobj, a whole keyword, that only white space separates from the end of , the + /// end-of-line before it left out; -1 when none does. + /// + /// The bytes from the start of the data to where the search for its endstream stopped. + public static int EndObjBefore(ReadOnlySpan stretch) + { + var end = stretch.LastIndexOfAnyExcept(PdfCharacters.Whitespace) + 1; + return EndObjEnding(stretch[..end], atDataStart: true); + } + + /// + /// Gives where the data ends when end with a whole endobj: where the keyword starts, the + /// end-of-line before it left out; -1 when they do not. + /// + /// Bytes that end where the white space before the stop starts. + /// Whether the bytes start where the data does, before which no byte belongs to it. + public static int EndObjEnding(ReadOnlySpan bytes, bool atDataStart) + { + if (!bytes.EndsWith(EndObjKeyword)) + { + return -1; + } + + var keyword = bytes.Length - EndObjKeyword.Length; + + // The keyword is whole: what precedes it ends a token, or is the start of the data. + if (keyword > 0 ? PdfCharacters.IsRegular(bytes[keyword - 1]) : !atDataStart) + { + return -1; + } + + var end = keyword; + + if (end > 0 && bytes[end - 1] == (byte)'\n') + { + end--; + } + + if (end > 0 && bytes[end - 1] == (byte)'\r') + { + end--; + } + + return end; + } + + /// + /// Searches one read of a search for endstream — the parser's bytes, or a window of the file — for the first + /// endstream, and for the first object header before it: returns where the data ends in the file, the + /// end-of-line before the keyword left out, or -1 when the read holds no endstream or a header comes first, + /// whose start in the file is then , -1 otherwise. + /// + /// + /// The overlap between reads puts the end-of-line before a keyword first seen in a read inside it, unless the keyword + /// starts the data. A cross-reference stream whose /Length the chain could not read is searched as + /// : its rows may spell a header, and may end in a carriage return, so neither stops nor + /// trims it (#220). + /// + /// The bytes read, after what the reads before carry into them. + /// Whether they end where the search does: at the next object the index places, or the end of the file. + /// Whether the stream's /Length is one the chain could not read, which is no fault. + /// Where the header that stopped the search starts in the file, or -1. + public static long SearchStretch(in HeaderText text, bool endsRange, bool deferred, out long header) + { + var span = text.Bytes; + var keyword = span.IndexOf(EndStreamKeyword); + header = deferred ? -1 : text.FindHeader(keyword >= 0 ? keyword : span.Length, endsRange); + + if (keyword < 0 || header >= 0) + { + return -1; + } + + if (keyword > 0 && span[keyword - 1] == (byte)'\n') + { + keyword--; + } + + if (!deferred && keyword > 0 && span[keyword - 1] == (byte)'\r') + { + keyword--; + } + + return text.PositionOf(keyword); + } +} diff --git a/tests/AdCodicem.Pdf.Tests/CorpusReadingTests.cs b/tests/AdCodicem.Pdf.Tests/CorpusReadingTests.cs index 2dd3e20e..645d07a1 100644 --- a/tests/AdCodicem.Pdf.Tests/CorpusReadingTests.cs +++ b/tests/AdCodicem.Pdf.Tests/CorpusReadingTests.cs @@ -248,9 +248,10 @@ public void A_document_holds_each_report_once_however_often_its_streams_are_deco [Fact] public void A_damaged_object_stream_s_fault_is_held_once_after_a_full_read_and_a_validation() { - // Object stream 417 of the PDFBox report turns corrupt where a block of zeros erased its data. Reading it serves its - // objects; reading object 5922 rebuilds the index, which decodes it again; the full read decodes it a third time, - // and the validator a fourth: its fault was held four times, and is held once (#144). + // Object stream 417 of the PDFBox report lost its end to a block of zeros: its data is the 1,302 bytes it declares, + // which no endstream follows before the next object (#174), and its Flate data ends before them. Reading it serves + // its objects; reading object 5922 rebuilds the index, which decodes it again; the full read decodes it a third + // time, and the validator a fourth: its fault was held four times, and is held once (#144). const string File = "remote/pdfbox/distiller6-zeroed-object-stream-pdfbox3947.pdf"; Assert.SkipUnless(Corpus.Paths.Contains(File), $"{File} is a remote document this machine did not fetch"); using var document = PdfDocument.Open(Corpus.PathOf(File), OptionsFor(Corpus.Get(File)) with { DiagnosticCapacity = int.MaxValue }); @@ -259,7 +260,7 @@ public void A_damaged_object_stream_s_fault_is_held_once_after_a_full_read_and_a new PdfValidator().Validate(document); document.Diagnostics.Where(d => d.Position == 1670741 && d.Code == PdfDiagnosticCodes.FilterFailed).Select(d => d.ToString()) - .Should().Equal("Warning filter.failed at 1670741: A Flate stream is corrupt at byte 33853 of its 34152; the 102785 bytes decoded before the fault was found were kept."); + .Should().Equal("Warning filter.failed at 1670741: A Flate stream ends before its data does; what decoded before the end was kept."); } [Theory] diff --git a/tests/AdCodicem.Pdf.Tests/FilterDamageTests.cs b/tests/AdCodicem.Pdf.Tests/FilterDamageTests.cs index 64d4f684..e02d7e8c 100644 --- a/tests/AdCodicem.Pdf.Tests/FilterDamageTests.cs +++ b/tests/AdCodicem.Pdf.Tests/FilterDamageTests.cs @@ -1233,9 +1233,10 @@ public void Reports_no_missing_end_marker_where_a_filter_stopped_at_the_bound_co [InlineData("own bound")] public void Reports_no_missing_greater_than_sign_where_the_reader_cut_ascii85_data_after_its_tilde(string cut) { - // The ~> closes the data, and the reader's cut falls between its two characters: the window MaxObjectLength allows - // ends on the ~, or the hex filter before stops at the bound there. Or the filter stops at its own bound in its - // last group, after the ~ was read. What lies past a cut is the guard's, and only the guard is reported (#199). + // The ~> closes the data, and the reader's cut falls between its two characters: the data MaxStreamLength lets the + // reader read ends on the ~, or the hex filter before stops at the bound there. Or the filter stops at its own bound + // in its last group, after the ~ was read. What lies past a cut is the guard's, and only the guard is reported + // (#199). const string Encoded = "87cURD]i,\"Ebo7~>"; var diagnostics = new PdfDiagnostics(); string[] expected; @@ -1245,15 +1246,13 @@ public void Reports_no_missing_greater_than_sign_where_the_reader_cut_ascii85_da var file = new TestPdfBuilder() .WithObject(1, "<< /Type /Catalog /Pages 2 0 R >>") .WithObject(2, "<< /Type /Pages /Kids [] /Count 0 >>") - .WithObject(3, $"<< /Filter /ASCII85Decode /Length 2 >>\nstream\n{Encoded}{new string(' ', 400)}\nendstream") + .Stream(3, "/Filter /ASCII85Decode", Encoded + new string(' ', 400)) .BuildClassic(rootNumber: 1); - var header = file.AsSpan().IndexOf("3 0 obj"u8); - var data = file.AsSpan().IndexOf(Encoding.ASCII.GetBytes(Encoded)); - var options = PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxObjectLength = data + Encoded.Length - 1 - header } }; + var options = PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxStreamLength = Encoded.Length - 1 } }; using var document = PdfDocument.Open(file, options); Encoding.ASCII.GetString(document.GetObject(new PdfObjectId(3)).AsStream().Required().Decode(diagnostics).Span).Should().Be("Hello World"); - expected = [PdfDiagnosticCodes.LimitObject]; + expected = [PdfDiagnosticCodes.LimitStreamLength]; document.Diagnostics.Select(d => d.Code).Concat(diagnostics.Select(d => d.Code)).Should().Equal(expected); return; } @@ -1334,18 +1333,16 @@ public void Reports_no_end_marker_a_guard_of_the_reader_cut_off_and_a_fault_in_w var file = new TestPdfBuilder() .WithObject(1, "<< /Type /Catalog /Pages 2 0 R >>") .WithObject(2, "<< /Type /Pages /Kids [] /Count 0 >>") - .WithObject(3, $"<< /Filter [/ASCIIHexDecode /{filter}] /Length 10 >>\nstream\n{hex}\nendstream") + .Stream(3, $"/Filter [/ASCIIHexDecode /{filter}]", hex) .BuildClassic(rootNumber: 1); - var header = file.AsSpan().IndexOf("3 0 obj"u8); - var data = file.AsSpan().IndexOf(Encoding.ASCII.GetBytes(hex)); - var options = PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxObjectLength = data + characters - header } }; + var options = PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxStreamLength = characters } }; using var document = PdfDocument.Open(file, options); document.GetObject(new PdfObjectId(3)).AsStream().Required().Decode(document.Diagnostics); string[] expected = stray && filter == "ASCII85Decode" - ? [PdfDiagnosticCodes.LimitObject, PdfDiagnosticCodes.FilterFailed] - : [PdfDiagnosticCodes.LimitObject]; + ? [PdfDiagnosticCodes.LimitStreamLength, PdfDiagnosticCodes.FilterFailed] + : [PdfDiagnosticCodes.LimitStreamLength]; document.Diagnostics.Select(d => d.Code).Should().Equal(expected); } @@ -1429,56 +1426,49 @@ public void Counts_the_bytes_a_damaged_stream_kept_before_its_predictor_takes_th [Fact] public void Reports_a_fault_in_the_part_of_a_stream_a_guard_of_the_reader_kept_and_counts_that_part() { - // A /Length far too short sends the reader looking for endstream, past the largest window MaxObjectLength - // allows, which ends 10,000 bytes into the stream's data, well past the byte its Flate data turns corrupt - // at. The guard is reported, and so is the fault, which lies in bytes the reader read and is the file's: - // counted in the data the Flate filter was given, which is what the window kept. + // MaxStreamLength lets the reader read 10,000 bytes of the stream's hex data, well past the byte its Flate data + // turns corrupt at. The guard is reported, and so is the fault, which lies in bytes the reader read and is the + // file's: counted in the data the Flate filter was given, which is what the guard kept. const int Kept = 10_000; var plain = ContentStream(200); var hex = Convert.ToHexString(CorruptAfter(plain, out var faultAt, after: RandomBytes(60_000))) + ">"; var file = new TestPdfBuilder() .WithObject(1, "<< /Type /Catalog /Pages 2 0 R >>") .WithObject(2, "<< /Type /Pages /Kids [] /Count 0 >>") - .WithObject(3, $"<< /Filter [/ASCIIHexDecode /FlateDecode] /Length 10 >>\nstream\n{hex}\nendstream") + .Stream(3, "/Filter [/ASCIIHexDecode /FlateDecode]", hex) .BuildClassic(rootNumber: 1); - var header = file.AsSpan().IndexOf("3 0 obj"u8); - var dataStart = file.AsSpan().IndexOf(Encoding.ASCII.GetBytes(hex)); - var options = PdfReaderOptions.Default with - { - Limits = PdfReaderLimits.Default with { MaxObjectLength = dataStart + (2 * Kept) - header }, - }; + var options = PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxStreamLength = 2 * Kept } }; faultAt.Should().BeLessThan(Kept); using var document = PdfDocument.Open(file, options); var decoded = document.GetObject(new PdfObjectId(3)).AsStream().Required().Decode(document.Diagnostics); decoded.ToArray().Should().Equal(plain); - document.Diagnostics.Select(d => d.Code).Should().Equal(PdfDiagnosticCodes.LimitObject, PdfDiagnosticCodes.FilterFailed); + document.Diagnostics.Select(d => d.Code).Should().Equal(PdfDiagnosticCodes.LimitStreamLength, PdfDiagnosticCodes.FilterFailed); document.Diagnostics[^1].Message.Should().Be(CorruptAt(faultAt, Kept, plain.Length)); } [Fact] public void Says_nothing_of_the_tail_a_guard_of_the_reader_cut_off() { - // A /Length far too short sends the reader looking for endstream, which lies past the largest window - // MaxObjectLength allows: the object is kept as far as the window went, and the guard is reported. - // That the Flate data then runs out is the reader's doing, not the file's. + // The stream's data runs past what MaxStreamLength lets the reader read: the data is kept as far as the guard + // allows, and the guard is reported. That the Flate data then runs out is the reader's doing, not the file's. var compressed = Compress(RandomBytes(100_000), CompressionLevel.Optimal); var hex = Convert.ToHexString(compressed) + ">"; var file = new TestPdfBuilder() .WithObject(1, "<< /Type /Catalog /Pages 2 0 R >>") .WithObject(2, "<< /Type /Pages /Kids [] /Count 0 >>") - .WithObject(3, $"<< /Filter [/ASCIIHexDecode /FlateDecode] /Length 10 >>\nstream\n{hex}\nendstream") + .Stream(3, "/Filter [/ASCIIHexDecode /FlateDecode]", hex) .BuildClassic(rootNumber: 1); - var options = PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxObjectLength = 64 * 1024 } }; + var options = PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxStreamLength = 64 * 1024 } }; using var cut = PdfDocument.Open(file, options); using var whole = PdfDocument.Open(file); cut.GetObject(new PdfObjectId(3)).AsStream().Required().Decode(cut.Diagnostics); whole.GetObject(new PdfObjectId(3)).AsStream().Required().Decode(whole.Diagnostics).Length.Should().Be(100_000); - cut.Diagnostics.Select(d => d.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); - whole.Diagnostics.Select(d => d.Code).Should().Equal(PdfDiagnosticCodes.StreamLengthInvalid); + cut.Diagnostics.Select(d => d.Code).Should().Equal(PdfDiagnosticCodes.LimitStreamLength); + whole.Diagnostics.Should().BeEmpty(); } [Theory] @@ -1562,27 +1552,24 @@ public void Drops_the_unit_a_filter_before_it_parted_at_the_bound_rather_than_co [InlineData("ASCII85Decode", 40)] public void Drops_the_unit_a_guard_of_the_reader_parted_rather_than_complete_it(string filter, int characters) { - // A /Length far too short sends the reader looking for endstream past the largest window MaxObjectLength - // allows, which ends after the given number of characters of the data: a digit without its pair, or a group - // short of five, is one the guard parted, and what comes back is the start of the data, with the guard alone + // MaxStreamLength lets the reader read the given number of characters of the data: a digit without its pair, or a + // group short of five, is one the guard parted, and what comes back is the start of the data, with the guard alone // reported. var plain = Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat("The quick brown fox. ", 20))); var encoded = Encoding.ASCII.GetString(Encode(plain, filter)); var file = new TestPdfBuilder() .WithObject(1, "<< /Type /Catalog /Pages 2 0 R >>") .WithObject(2, "<< /Type /Pages /Kids [] /Count 0 >>") - .WithObject(3, $"<< /Filter /{filter} /Length 10 >>\nstream\n{encoded}\nendstream") + .Stream(3, $"/Filter /{filter}", encoded) .BuildClassic(rootNumber: 1); - var header = file.AsSpan().IndexOf("3 0 obj"u8); - var data = file.AsSpan().IndexOf(Encoding.ASCII.GetBytes(encoded)); - var options = PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxObjectLength = data + characters - header } }; + var options = PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxStreamLength = characters } }; using var document = PdfDocument.Open(file, options); var decoded = document.GetObject(new PdfObjectId(3)).AsStream().Required().Decode(document.Diagnostics); var whole = filter == "ASCIIHexDecode" ? characters / 2 : characters / 5 * 4; decoded.ToArray().Should().Equal(plain.AsSpan(0, whole).ToArray()); - document.Diagnostics.Select(d => d.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.Diagnostics.Select(d => d.Code).Should().Equal(PdfDiagnosticCodes.LimitStreamLength); } [Theory] @@ -1590,23 +1577,21 @@ public void Drops_the_unit_a_guard_of_the_reader_parted_rather_than_complete_it( [InlineData("ASCII85Decode", "87cURD]i,\"Ebo7~>", "Hello World")] public void Completes_the_unit_its_end_marker_closes_however_the_reader_cut_the_data_after_it(string filter, string encoded, string expected) { - // The window MaxObjectLength allows ends a few characters past the end marker: the digit, or the short group, - // before the marker is the file's last, which the marker closes, and is completed as in data nothing cut. + // The data MaxStreamLength lets the reader read ends a few characters past the end marker: the digit, or the short + // group, before the marker is the file's last, which the marker closes, and is completed as in data nothing cut. var data = encoded + new string(' ', 400); var file = new TestPdfBuilder() .WithObject(1, "<< /Type /Catalog /Pages 2 0 R >>") .WithObject(2, "<< /Type /Pages /Kids [] /Count 0 >>") - .WithObject(3, $"<< /Filter /{filter} /Length 2 >>\nstream\n{data}\nendstream") + .Stream(3, $"/Filter /{filter}", data) .BuildClassic(rootNumber: 1); - var header = file.AsSpan().IndexOf("3 0 obj"u8); - var start = file.AsSpan().IndexOf(Encoding.ASCII.GetBytes(data)); - var options = PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxObjectLength = start + encoded.Length + 5 - header } }; + var options = PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxStreamLength = encoded.Length + 5 } }; using var document = PdfDocument.Open(file, options); var decoded = document.GetObject(new PdfObjectId(3)).AsStream().Required().Decode(document.Diagnostics); Encoding.ASCII.GetString(decoded.Span).Should().Be(expected); - document.Diagnostics.Select(d => d.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.Diagnostics.Select(d => d.Code).Should().Equal(PdfDiagnosticCodes.LimitStreamLength); } [Fact] @@ -1678,7 +1663,7 @@ public void Says_nothing_of_the_end_of_data_a_filter_before_it_stopped_at_the_bo [InlineData(100)] public void Says_nothing_of_a_guard_s_cut_that_falls_in_the_checksum_or_after_the_data(int bytesPastTheBody) { - // The largest window MaxObjectLength allows ends 2 bytes into the checksum, or in the white space after + // The data MaxStreamLength lets the reader read ends 2 bytes into the checksum, or in the white space after // the whole zlib stream: a checksum the reader cut is no repair of the file's, and data that ended // before the cut is whole. Only the guard is reported. var compressed = Compress(ContentStream(200), CompressionLevel.Optimal); @@ -1686,19 +1671,17 @@ public void Says_nothing_of_a_guard_s_cut_that_falls_in_the_checksum_or_after_th var file = new TestPdfBuilder() .WithObject(1, "<< /Type /Catalog /Pages 2 0 R >>") .WithObject(2, "<< /Type /Pages /Kids [] /Count 0 >>") - .WithObject(3, $"<< /Filter [/ASCIIHexDecode /FlateDecode] /Length 10 >>\nstream\n{hex}{new string(' ', 400)}>\nendstream") + .Stream(3, "/Filter [/ASCIIHexDecode /FlateDecode]", hex + new string(' ', 400) + ">") .BuildClassic(rootNumber: 1); - var header = file.AsSpan().IndexOf("3 0 obj"u8); - var data = file.AsSpan().IndexOf(Encoding.ASCII.GetBytes(hex)); - var bodyEnd = data + (2 * (compressed.Length - ChecksumLength)); - var cut = bytesPastTheBody < 0 ? bodyEnd + (2 * -bytesPastTheBody) : data + hex.Length + bytesPastTheBody; - var options = PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxObjectLength = cut - header } }; + var bodyEnd = 2 * (compressed.Length - ChecksumLength); + var cut = bytesPastTheBody < 0 ? bodyEnd + (2 * -bytesPastTheBody) : hex.Length + bytesPastTheBody; + var options = PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxStreamLength = cut } }; using var document = PdfDocument.Open(file, options); var decoded = document.GetObject(new PdfObjectId(3)).AsStream().Required().Decode(document.Diagnostics); decoded.Length.Should().Be(ContentStream(200).Length); - document.Diagnostics.Select(d => d.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.Diagnostics.Select(d => d.Code).Should().Equal(PdfDiagnosticCodes.LimitStreamLength); } [Theory] diff --git a/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs b/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs index a532bae0..7f07b965 100644 --- a/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs +++ b/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs @@ -169,6 +169,113 @@ public void An_object_stream_header_s_tables_start_no_larger_than_its_bytes_can_ allocated.Should().BeLessThanOrEqualTo(2 * (sizeof(int) + 32), "two tables of one pair each"); } + [Theory] + [InlineData(0)] + [InlineData(100_000)] + [InlineData(1_000_000)] + public void A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard(int dataLength) + { + // "/Length 2147483647", the data, its endstream, then an object of 17 MB: the window grew to MaxObjectLength to + // settle a length the file cannot hold, read 21.5 MB, reported limit.object and marked the object cut, though its + // data was found whole, and threw under ThrowOnLimit (#174). The endstream found settles it, the data read once. + var data = dataLength == 0 ? "short" : new string('d', dataLength); + var file = new TestPdfBuilder() + .WithObject(1, "<< /Type /Catalog /Pages 2 0 R >>") + .WithObject(2, "<< /Type /Pages /Kids [] /Count 0 >>") + .WithObject(5, "<< /Length 2147483647 >>\nstream\n" + data + "\nendstream") + .WithObject(6, "(" + new string('p', 17 * 1024 * 1024) + ")") + .BuildClassic(rootNumber: 1); + + foreach (var throwOnLimit in new[] { false, true }) + { + using var source = new StrictCountingSource(file); + using var document = PdfDocument.Open(source, new PdfReaderOptions { ThrowOnLimit = throwOnLimit }, ownsSource: false); + var opened = source.BytesRead; + var stream = document.GetObject(new PdfObjectId(5)).AsStream().Required(); + + stream.Data.Length.Should().Be(data.Length); + document.Diagnostics.Should().ContainSingle().Which.Message.Should().Be( + string.Create(CultureInfo.InvariantCulture, $"The stream declared 2147483647 bytes but ended after {data.Length}.")); + document.Reader.IsCutAtLimit(5).Should().BeFalse(); + (source.BytesRead - opened).Should().BeLessThan(data.Length + (4L * PdfFileReader.InitialObjectWindow), "the data, read once"); + } + } + + [Fact] + public void Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each() + { + // Fifty such streams before an object of 17 MB: each grew its window to MaxObjectLength, and the reader read the + // file sixty times over (#174). + const int Streams = 50; + var builder = new TestPdfBuilder() + .WithObject(1, "<< /Type /Catalog /Pages 2 0 R >>") + .WithObject(2, "<< /Type /Pages /Kids [] /Count 0 >>"); + + for (var k = 0; k < Streams; k++) + { + builder.WithObject(10 + k, "<< /Length 2147483647 >>\nstream\nshort\nendstream"); + } + + var file = builder.WithObject(100, "(" + new string('p', 17 * 1024 * 1024) + ")").BuildClassic(rootNumber: 1); + using var source = new StrictCountingSource(file); + using var document = PdfDocument.Open(source, options: null, ownsSource: false); + var opened = source.BytesRead; + + for (var k = 0; k < Streams; k++) + { + document.GetObject(new PdfObjectId(10 + k)).AsStream().Required().Data.Length.Should().Be(5); + } + + (source.BytesRead - opened).Should().BeLessThan(Streams * 2L * PdfFileReader.InitialObjectWindow, "a window and a few bytes for each"); + document.Diagnostics.Should().HaveCount(Streams).And.OnlyContain(diagnostic => diagnostic.Code == PdfDiagnosticCodes.StreamLengthInvalid); + } + + [Fact] + public void A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows() + { + // iPRES's t02-05-01-014 before an object of 20 MB: no endstream after the data, which the window held, so it grew to + // MaxObjectLength looking for one, took 16 MB as the data of a 68-byte stream, reported limit.object and marked the + // object cut (#174). Its endobj plays the endstream's part. + var data = new string('d', 68); + var file = new TestPdfBuilder() + .WithObject(1, "<< /Type /Catalog /Pages 2 0 R >>") + .WithObject(2, "<< /Type /Pages /Kids [] /Count 0 >>") + .WithObject(5, "<< /Length 68 >>\nstream\n" + data + "\r\n\n") + .WithObject(6, "(" + new string('p', 20 * 1024 * 1024) + ")") + .BuildClassic(rootNumber: 1); + + using var source = new StrictCountingSource(file); + using var document = PdfDocument.Open(source, options: null, ownsSource: false); + var opened = source.BytesRead; + + document.GetObject(new PdfObjectId(5)).AsStream().Required().Data.Length.Should().Be(68); + + (source.BytesRead - opened).Should().BeLessThanOrEqualTo(PdfFileReader.InitialObjectWindow, "the window holds the object and what follows it"); + document.Diagnostics.Should().ContainSingle().Which.Code.Should().Be(PdfDiagnosticCodes.StreamTruncated); + document.Reader.IsCutAtLimit(5).Should().BeFalse(); + } + + [Fact] + public void A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard() + { + // "/Length 2147483647" and 20 MB of data to the end of the file, with no endstream and no endobj: the window grew to + // MaxObjectLength and the guard was reported (#174). The data runs to the end of the file, the object without its + // endobj, as for any stream the end of the file cuts. + var text = "%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n" + + "5 0 obj\n<< /Length 2147483647 >>\nstream\n"; + var file = Encoding.ASCII.GetBytes(text + new string('d', 20 * 1024 * 1024)); + + using var document = PdfDocument.Open(file); + var stream = document.GetObject(new PdfObjectId(5)).AsStream().Required(); + + stream.Data.Length.Should().Be(20 * 1024 * 1024); + document.Diagnostics.Should().ContainSingle(d => d.Code == PdfDiagnosticCodes.StreamTruncated).Which.Message.Should().Be( + "The stream declared 2147483647 bytes, and no endstream follows its data before the end of the file; the 20971520 bytes to the end of the file are taken as its data."); + document.Diagnostics.Should().NotContain(d => d.Code.StartsWith("limit.", StringComparison.Ordinal)); + document.Reader.IsCutAtLimit(5).Should().BeFalse(); + document.Reader.IsEndObjMissing(5, out _).Should().BeTrue(); + } + [Fact] public void Reading_streams_whose_data_share_a_stretch_of_a_file_held_in_memory_allocates_none_of_their_data() { diff --git a/tests/AdCodicem.Pdf.Tests/ParserTests.cs b/tests/AdCodicem.Pdf.Tests/ParserTests.cs index 6aa17f1e..e48facfd 100644 --- a/tests/AdCodicem.Pdf.Tests/ParserTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ParserTests.cs @@ -182,12 +182,26 @@ public void Recovers_a_stream_whose_declared_length_is_wrong() } [Theory] - [InlineData(0, true)] - [InlineData(-1, false)] - [InlineData(11, false)] - public void Looks_for_endstream_only_inside_the_bytes_it_is_given(long position, bool expected) + [InlineData("\nendstream\n", "EndStream")] + [InlineData(" \n %a comment\r\n\t \fendstream endobj", "EndStream")] + [InlineData("\nendstream", "Undecided")] + [InlineData("x\nendstream", "None")] + [InlineData("\nendstrean", "None")] + [InlineData(" ", "Undecided")] + [InlineData("\nendst", "Undecided")] + [InlineData("\n%a comment the bytes end inside", "Undecided")] + public void Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell(string bytes, string expected) { - PdfObjectParser.IsEndStreamAt("\nendstream"u8, position).Should().Be(expected); + StreamEnds.ReadEnd(Encoding.ASCII.GetBytes(bytes), reachesEndOfFile: false).Kind.ToString().Should().Be(expected); + } + + [Theory] + [InlineData(" ")] + [InlineData("\nendst")] + [InlineData("\n%a comment the file ends inside")] + public void Bytes_that_end_the_file_before_endstream_hold_none(string bytes) + { + StreamEnds.ReadEnd(Encoding.ASCII.GetBytes(bytes), reachesEndOfFile: true).Kind.Should().Be(StreamEndKind.None); } [Fact] @@ -200,6 +214,38 @@ public void Recovers_a_stream_that_never_ends() diagnostics.Contains(PdfDiagnosticCodes.StreamTruncated).Should().BeTrue(); } + [Fact] + public void A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object() + { + // The parser alone stops the search where the reader does, at the first header its bytes hold, and the object's + // endobj before it ends the data (#174). + var diagnostics = new PdfDiagnostics(); + var stream = ParseStream("<< /Length 3 >>\nstream\nhello\nendobj\n6 0 obj\n<< /Length 5 >>\nstream\nworld\nendstream\nendobj", diagnostics); + + Encoding.ASCII.GetString(stream.GetRawBytes().Span).Should().Be("hello"); + diagnostics.Should().ContainSingle().Which.Code.Should().Be(PdfDiagnosticCodes.StreamTruncated); + } + + [Fact] + public void A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it() + { + var diagnostics = new PdfDiagnostics(); + var stream = ParseStream("<< /Length 99 >>\nstream\nhello\n6 0 obj\n<< /Length 5 >>\nstream\nworld\nendstream\nendobj", diagnostics); + + Encoding.ASCII.GetString(stream.GetRawBytes().Span).Should().Be("hello\n"); + diagnostics.Should().ContainSingle().Which.Code.Should().Be(PdfDiagnosticCodes.StreamLengthInvalid); + } + + [Fact] + public void A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more() + { + // No endstream, no header, no endobj in the window: a larger one may hold them. + var parser = new PdfObjectParser(Encoding.ASCII.GetBytes("<< /Length 300 >>\nstream\n" + new string('d', 100)), endsData: false); + + parser.ParseObject().Should().BeOfType(); + parser.IsTruncated.Should().BeTrue(); + } + [Fact] public void Resolves_a_length_given_as_an_indirect_reference() { diff --git a/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs b/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs index 374cb54b..0a87176c 100644 --- a/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs @@ -1713,9 +1713,9 @@ public void A_rebuild_that_fills_the_index_keeps_no_member_a_guard_hides_past_it [InlineData("<< /Type /ObjStm /N 0 /First 4 >>")] public void An_object_stream_refused_for_its_own_fault_hides_no_member_where_a_limit_also_cut_its_data(string dictionary) { - // N2-02: object stream 10's dictionary, read whole, gives no usable /N; its data, with no /Length, runs past the - // limit. The stream is refused for the file's fault, and the catalog it holds was taken for one the guard cut: no - // search for catalog 5, file.root-invalid silent, and the validator's probe said the stream was not checked. + // N2-02: object stream 10's dictionary, read whole, gives no usable /N; its data runs past what MaxStreamLength lets + // the reader read. The stream is refused for the file's fault, and the catalog it holds was taken for one the guard + // cut: no search for catalog 5, file.root-invalid silent, and the validator's probe said the stream was not checked. var file = new HandWrittenFile(); var rows = new Dictionary { @@ -1726,9 +1726,10 @@ public void An_object_stream_refused_for_its_own_fault_hides_no_member_where_a_l [1] = (2, 10, 0), }; file.CrossReferenceStream(11, 12, rows, "/Root 1 0 R"); - using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = 256 } }); + using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxStreamLength = 256 } }); - document.Reader.IsCutAtLimit(10).Should().BeTrue(); + document.GetObject(new PdfObjectId(10)).AsStream().Required().GetRawBytes().Length.Should().Be(256); + document.Diagnostics.Should().Contain(entry => entry.Code == PdfDiagnosticCodes.LimitStreamLength); document.Reader.IsCutAtLimit(1).Should().BeFalse(); document.Reader.Structure.RootCutByGuard.Should().BeFalse(); document.Reader.CatalogFoundAs.Should().Be(5); diff --git a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs index ba886f09..36e01bfc 100644 --- a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs +++ b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs @@ -104,9 +104,11 @@ public void A_stream_past_the_window_whose_length_is_long_ends_at_the_endstream_ } [Fact] - public void A_stream_past_the_window_with_no_endstream_before_the_next_object_keeps_its_declared_length() + public void A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj() { - // The endstream the search would find next is object 6's: the next object bounds it. + // The endstream the search would find next is object 6's: the next object bounds it. No endstream before it, the + // object's own endobj ends the data, its end-of-line left out, rather than a declared length that runs on into + // object 6 (#174). var data = Data(3 * Window); var file = Reachable((StreamNumber, Stream(data.Length + 100, data, end: "\n")), (6, Stream(5, "hello"))); var dataStart = DataStartOf(file, StreamNumber); @@ -115,22 +117,24 @@ public void A_stream_past_the_window_with_no_endstream_before_the_next_object_ke using var document = PdfDocument.Open(file); var stream = document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required(); - stream.Data.Length.Should().Be(data.Length + 100); - document.Diagnostics.Should().ContainSingle().Which.Message.Should().Be( - Invariant($"The stream declared {data.Length + 100} bytes, and no endstream follows them before the next object, at {next}; the declared length is kept.")); + stream.Data.Length.Should().Be(data.Length + 1); + var diagnostic = document.Diagnostics.Should().ContainSingle().Which; + diagnostic.Code.Should().Be(PdfDiagnosticCodes.StreamTruncated); + diagnostic.Message.Should().Be( + Invariant($"The stream declared {data.Length + 100} bytes, and no endstream follows its data before the next object, at {next}: its object's endobj ends it, after {data.Length + 1} bytes.")); document.Reader.TryGetStreamLengthFault(StreamNumber, out var fault).Should().BeTrue(); fault.Should().Be(new StreamLengthFault { Form = StreamLengthForm.Integer, Declared = data.Length + 100, - Taken = data.Length + 100, - EndStream = EndStreamState.MissingBeforeNextObject, + Taken = data.Length + 1, + EndStream = EndStreamState.MissingBeforeEndObj, NextObject = next, DataStart = dataStart, }); - // Where the object ends is unknown, and its endobj is not judged. + // The object ends with its own endobj. document.Reader.IsEndObjMissing(StreamNumber, out _).Should().BeFalse(); new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.ObjectEndObjMissing).Should().BeFalse(); } @@ -153,20 +157,21 @@ public void A_stream_past_the_window_with_no_endstream_before_the_end_of_the_fil } [Fact] - public void A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_keeps_its_declared_length() + public void A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched() { - // The next object starts inside the window: the part of the window before it is all the search reads. + // The next object starts inside the window: the part of the window before it is all the search reads, and the + // object's endobj before it ends the data (#174). var file = Document((StreamNumber, Stream(3 * Window, "short data", end: string.Empty)), (6, "(" + new string('p', 4 * Window) + ")")); using var source = new StrictCountingSource(file); using var document = PdfDocument.Open(source, options: null, ownsSource: false); var beforeLoading = source.BytesRead; - document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required().Data.Length.Should().Be(3 * Window); + document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required().Data.Length.Should().Be("short data".Length); (source.BytesRead - beforeLoading).Should().Be(Window + PdfObjectParser.EndObjLookahead, "the file past the window is not searched"); document.Diagnostics.Should().ContainSingle().Which.Message.Should().Be( - Invariant($"The stream declared {3 * Window} bytes, and no endstream follows them before the next object, at {HeaderOf(file, 6)}; the declared length is kept.")); + Invariant($"The stream declared {3 * Window} bytes, and no endstream follows its data before the next object, at {HeaderOf(file, 6)}: its object's endobj ends it, after 10 bytes.")); } [Fact] @@ -186,6 +191,108 @@ public void A_stream_whose_declared_length_ends_the_file_without_endstream_keeps document.Reader.IsEndObjMissing(StreamNumber, out _).Should().BeFalse("where the object ends is unknown"); } + [Theory] + [InlineData(" ", false)] + [InlineData(" ", true)] + [InlineData("\r\n% a comment between the data and its keyword\r\n", false)] + [InlineData("\r\n% a comment between the data and its keyword\r\n", true)] + [InlineData("\t\f\0 \r\n ", false)] + [InlineData("\t\f\0 \r\n ", true)] + public void A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed(string gap, bool pastWindow) + { + // ISO 32000-1 (7.3.8.1) only says an end-of-line should precede endstream, and 7.2.3 makes a comment white space: + // five spaces, or a comment, after a length that was right made it a wrong one, the gap taken into the data, and + // the file past the window searched (#174). Past the window, the bytes after the declared length are all it reads. + var data = Data(pastWindow ? 3 * Window : 100); + var file = Document((StreamNumber, Stream(data.Length, data, end: gap + "endstream")), (6, "(six)")); + + using var source = new StrictCountingSource(file); + using var document = PdfDocument.Open(source, options: null, ownsSource: false); + var beforeLoading = source.BytesRead; + var stream = document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required(); + + stream.Data.Length.Should().Be(data.Length); + document.Diagnostics.Should().BeEmpty(); + document.Reader.TryGetStreamLengthFault(StreamNumber, out _).Should().BeFalse(); + document.Reader.IsEndObjMissing(StreamNumber, out _).Should().BeFalse(); + + if (pastWindow) + { + (source.BytesRead - beforeLoading).Should().BeLessThanOrEqualTo(Window + (2 * PdfObjectParser.EndObjLookahead) + 8, "the window, then the bytes after the declared length"); + } + } + + [Theory] + [InlineData(3 * 8192, 100)] + [InlineData(3 * 8192, 5_000)] + [InlineData(3 * 8192, 200_000)] + [InlineData(8192 - 200, 300)] + public void White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length(int length, int spaces) + { + // White space after the data longer than the reader reads at first, past the window or across its edge: the file is + // read on as far as it runs, and the endstream after it confirms the length (#174). + var data = Data(length); + var file = Document((StreamNumber, Stream(data.Length, data, end: new string(' ', spaces) + "endstream")), (6, "(six)")); + + using var document = PdfDocument.Open(file); + document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required().Data.Length.Should().Be(data.Length); + + document.Diagnostics.Should().BeEmpty(); + document.Reader.IsEndObjMissing(StreamNumber, out _).Should().BeFalse(); + } + + [Theory] + [InlineData("<< /Length 3 >>", false)] + [InlineData("<< /Length 3 >>", true)] + [InlineData("<< /Length -1 >>", false)] + [InlineData("<< /Length 2147483647 >>", false)] + [InlineData("<< /Length 2147483647 >>", true)] + [InlineData("<< /Length 20000 >>", false)] + [InlineData("<< /Length 9 0 R >>", false)] + [InlineData("<< >>", false)] + [InlineData("<< >>", true)] + public void A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it(string dictionary, bool pastWindow) + { + // Object 5 lost its endstream, and object 6, a stream, follows it. The search in the window took object 5's endobj, + // object 6's header and its data up to its endstream — 49 bytes for "hello" —, and a declared length past it was + // kept, whatever the search found. The search stops at object 6, the next object, and object 5's own endobj ends + // its data, its end-of-line left out, wherever its /Length points (#174). + var data = pastWindow ? Data(3 * Window) : "hello"; + var file = Document( + (StreamNumber, dictionary + "\nstream\n" + data), + (6, Stream(5, "world")), + (7, "(" + new string('p', 30_000) + ")")); + + using var document = PdfDocument.Open(file); + var stream = document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required(); + + Encoding.ASCII.GetString(stream.GetRawBytes().Span).Should().Be(data); + var diagnostic = document.Diagnostics.Should().ContainSingle().Which; + diagnostic.Code.Should().Be(PdfDiagnosticCodes.StreamTruncated); + diagnostic.Message.Should().EndWith(Invariant($"before the next object, at {HeaderOf(file, 6)}: its object's endobj ends it, after {data.Length} bytes.")); + document.Reader.IsEndObjMissing(StreamNumber, out _).Should().BeFalse("the object ends with its own endobj"); + + Encoding.ASCII.GetString(document.GetObject(new PdfObjectId(6)).AsStream().Required().GetRawBytes().Span).Should().Be("world"); + document.Diagnostics.Should().ContainSingle("object 6 is read alone, whole"); + } + + [Fact] + public void A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj() + { + // No endstream, no endobj: the data runs to the next object's header, which no declared length passes, and the + // object ends without its endobj (#174). + var data = Data(3 * Window); + var file = Document((StreamNumber, Stream(5 * Window, data, end: "\n7 0 obj\n(seven)")), (6, "(six)")); + var seven = Encoding.Latin1.GetString(file).IndexOf("\n7 0 obj", StringComparison.Ordinal) + 1; + + using var document = PdfDocument.Open(file); + document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required().Data.Length.Should().Be(data.Length + 1); + + document.Diagnostics.Should().ContainSingle().Which.Message.Should().Be( + Invariant($"The stream declared {5 * Window} bytes, and no endstream follows its data before the next object, at {seven}; the {data.Length + 1} bytes up to that object are taken as its data.")); + document.Reader.IsEndObjMissing(StreamNumber, out _).Should().BeTrue(); + } + [Fact] public void The_search_stops_at_an_object_the_index_as_written_places_though_a_rebuilt_index_lacks_it() { @@ -218,7 +325,8 @@ public void The_search_stops_at_an_object_the_index_as_written_places_though_a_r [Fact] public void The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened() { - // No index was written: the one the rebuild made is the only one, and object 6's endstream is past its header. + // No index was written: the one the rebuild made is the only one, and object 6's endstream is past its header. The + // object's endobj before it ends the data (#174). var data = Data(3 * Window); var file = new TestPdfBuilder() .WithObject(1, "<< /Type /Catalog /Pages 2 0 R >>") @@ -231,9 +339,9 @@ public void The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_docum document.WasRepaired.Should().BeTrue(); var stream = document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required(); - stream.Data.Length.Should().Be(data.Length - 7); - document.Diagnostics.Where(d => d.Code == PdfDiagnosticCodes.StreamLengthInvalid).Should().ContainSingle() - .Which.Message.Should().EndWith(Invariant($"before the next object, at {HeaderOf(file, 6)}; the declared length is kept.")); + stream.Data.Length.Should().Be(data.Length + 1); + document.Diagnostics.Where(d => d.Code == PdfDiagnosticCodes.StreamTruncated).Should().ContainSingle() + .Which.Message.Should().EndWith(Invariant($"before the next object, at {HeaderOf(file, 6)}: its object's endobj ends it, after {data.Length + 1} bytes.")); } [Fact] @@ -241,7 +349,7 @@ public void The_search_stops_at_an_object_header_the_bytes_hold_before_the_next_ { // Object 5 lost its endstream, and object 12's header follows it, an object no index places: the index as written // places object 6 next, past object 12's endstream. The header the bytes hold is the nearer, and the search stops - // there rather than take in object 12. + // there rather than take in object 12; object 5's endobj before it ends the data (#174). var data = Data(3 * Window); var file = Document( (StreamNumber, Stream(data.Length + 400, data, end: "\nendstreax\nendobj\n12 0 obj\n<< /Length 5 >>\nstream\nhello\nendstream")), @@ -252,20 +360,20 @@ public void The_search_stops_at_an_object_header_the_bytes_hold_before_the_next_ using var document = PdfDocument.Open(file); var stream = document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required(); - stream.Data.Length.Should().Be(data.Length + 400); + stream.Data.Length.Should().Be(data.Length + "\nendstreax".Length); HeaderOf(file, 6).Should().BeGreaterThan(twelve); document.Reader.TryGetStreamLengthFault(StreamNumber, out var fault).Should().BeTrue(); fault.Should().Be(new StreamLengthFault { Form = StreamLengthForm.Integer, Declared = data.Length + 400, - Taken = data.Length + 400, - EndStream = EndStreamState.MissingBeforeNextObject, + Taken = data.Length + "\nendstreax".Length, + EndStream = EndStreamState.MissingBeforeEndObj, NextObject = twelve, DataStart = dataStart, }); document.Diagnostics.Should().ContainSingle().Which.Message.Should().Be( - Invariant($"The stream declared {data.Length + 400} bytes, and no endstream follows them before the next object, at {twelve}; the declared length is kept.")); + Invariant($"The stream declared {data.Length + 400} bytes, and no endstream follows its data before the next object, at {twelve}: its object's endobj ends it, after {data.Length + "\nendstreax".Length} bytes.")); } [Theory] @@ -304,7 +412,8 @@ public void Text_that_follows_a_lost_endstream_stops_the_search_when_it_is_an_ob { // An object header is "N G obj" at a token boundary, as the bytes hold it: a number and a generation whose values // the parser takes, however many zeros lead them, and white space between its tokens, however much. Text that is - // one stops the search before the endstream after it, and the declared length is kept; text that only looks like + // one stops the search before the endstream after it, and the data, which no declared length runs past, ends there + // (#174); text that only looks like // one — endobj, obj glued to what follows or precedes it, digits after a regular character, a number or a // generation out of range, a comment between its tokens — is read past, and the endstream after it ends the data. var data = Data(3 * Window); @@ -321,11 +430,11 @@ public void Text_that_follows_a_lost_endstream_stops_the_search_when_it_is_an_ob if (header is { } at) { var start = dataStart + data.Length + 1 + at; - stream.Data.Length.Should().Be(data.Length + 400); + stream.Data.Length.Should().Be(data.Length + 1 + at); fault.EndStream.Should().Be(EndStreamState.MissingBeforeNextObject); fault.NextObject.Should().Be(start); document.Diagnostics.Should().ContainSingle().Which.Message.Should().Be( - Invariant($"The stream declared {data.Length + 400} bytes, and no endstream follows them before the next object, at {start}; the declared length is kept.")); + Invariant($"The stream declared {data.Length + 400} bytes, and no endstream follows its data before the next object, at {start}; the {data.Length + 1 + at} bytes up to that object are taken as its data.")); } else { @@ -357,7 +466,7 @@ public void The_start_and_the_end_of_the_search_bound_an_object_header_as_white_ } using var document = PdfDocument.Open(file); - document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required().Data.Length.Should().Be(data.Length + 400); + document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required().Data.Length.Should().Be((int)(header - dataStart), "the data runs to the header"); document.Reader.TryGetStreamLengthFault(StreamNumber, out var fault).Should().BeTrue(); fault.NextObject.Should().Be(header); @@ -378,7 +487,7 @@ public void A_header_the_edge_between_two_reads_of_the_search_cuts_is_decided_wh // the same with a regular character before or after it, or a number or a generation out of range, which are none. // What a read's start cuts of a header, the read before carries into it. The start of a read is no boundary before // a number, nor its end one after obj, unless the search starts or ends there. A header missed, the endstream after - // it would end the data; text taken for one, the declared length would be kept. + // it would end the data; text taken for one, the data would end there. var prefix = Invariant($"{StreamNumber} 0 obj\n") + Stream(0, string.Empty, end: string.Empty, width: 7); var read = PdfFileReader.EndStreamSearchFirstRead(Window - prefix.Length); var first = Window - PdfFileReader.EndStreamSearchOverlap + read; @@ -402,7 +511,7 @@ public void A_header_the_edge_between_two_reads_of_the_search_cuts_is_decided_wh if (header) { - stream.Data.Length.Should().Be(length + 400, because); + stream.Data.Length.Should().Be(length + 1, because); fault.NextObject.Should().Be(HeaderOf(file, StreamNumber) + at, because); } else @@ -443,7 +552,7 @@ public void A_header_whose_runs_span_whole_reads_of_the_search_is_decided_as_its if (header) { - stream.Data.Length.Should().Be(declared); + stream.Data.Length.Should().Be(data.Length + 1, "the data runs to the header"); fault.NextObject.Should().Be(dataStart + data.Length + 1, "the header starts where its number does, its zeros with it"); } else @@ -475,7 +584,8 @@ public void Streams_only_a_rebuilt_index_places_stop_at_the_next_header_and_hold // all, declares 100 bytes more than its data, which its endstream follows. Bounded by the index as written alone, // each of 10 to 38 ran to object 39's endstream: the searches read the file over and over and spent the document's // budget, object 50 was searched only when read before the rebuild, and it held 30,000 bytes or 30,100 according - // to the order. Each stream now stops at the header of the one after it, and holds the same data in either order, + // to the order. Each stream now stops at the header of the one after it, ends at its own endobj before that header + // (#174), and holds the same data in either order, // however the headers are written that both the rebuild and the parser take: more white space than a read of the // search repeats, zeros leading the generation. The searches cannot know where that header is before they read // it: reading the file 64 KB at a time, or 16 KB at first whatever window the object was parsed through, the @@ -521,9 +631,9 @@ public void Streams_only_a_rebuilt_index_places_stop_at_the_next_header_and_hold for (var number = 10; number < 39; number++) { - document.GetObject(new PdfObjectId(number)).AsStream().Required().Data.Length.Should().Be(data.Length + 100); + document.GetObject(new PdfObjectId(number)).AsStream().Required().Data.Length.Should().Be(data.Length); document.Reader.TryGetStreamLengthFault(number, out var fault).Should().BeTrue(); - fault.EndStream.Should().Be(EndStreamState.MissingBeforeNextObject, "the document's searches have read far less than they may"); + fault.EndStream.Should().Be(EndStreamState.MissingBeforeEndObj, "the document's searches have read far less than they may"); fault.NextObject.Should().Be(text.IndexOf("\n" + Header(header, number + 1) + "\n", StringComparison.Ordinal) + 1); } @@ -623,16 +733,20 @@ 0 6 new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.ObjectEndObjMissing).Should().Be(missing); } - [Fact] - public void What_follows_the_endstream_of_a_stream_past_the_window_is_unseen_past_a_few_bytes() + [Theory] + [InlineData("\nendobj\n", false)] + [InlineData("\n8 0 obj\n", true)] + public void What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs(string next, bool missing) { + // Past a few bytes of white space after endstream, what followed went unseen, and the object's endobj unjudged: the + // file is read on as far as white space runs (#174). var data = Data(3 * Window); - var file = Document((StreamNumber, Stream(data.Length, data, end: "\nendstream" + new string(' ', PdfObjectParser.EndObjLookahead)))); + var file = Document((StreamNumber, Stream(data.Length, data, end: "\nendstream" + new string(' ', 200) + next + "(unwritten)"))); using var document = PdfDocument.Open(file); - document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required(); + document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required().Data.Length.Should().Be(data.Length); - document.Reader.IsEndObjMissing(StreamNumber, out _).Should().BeFalse("what follows is not seen, and not judged"); + document.Reader.IsEndObjMissing(StreamNumber, out _).Should().Be(missing); } [Fact] @@ -969,9 +1083,39 @@ public void A_parser_without_diagnostics_reads_a_stream_whose_length_is_wrong_to [Theory] [InlineData(false)] [InlineData(true)] - public void A_stream_that_endobj_follows_without_its_endstream_says_what_it_takes(bool pastWindow) + public void A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length(bool pastWindow) + { + // iPRES's t02-05-01-014: its /Length is right, and its object's endobj follows the data without an endstream. The data + // ran to the end of the file, through object 6, the table and the trailer; the endobj plays the endstream's part now, + // and the declared length is kept, in the window or past it (#174). + var data = Data(pastWindow ? 3 * Window : 68); + var file = Encoding.ASCII.GetBytes( + "%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n" + + Invariant($"5 0 obj\n<< /Length {data.Length} >>\nstream\n{data}\r\n\n\nendobj \n6 0 obj\n(six)\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF\n")); + var dataStart = DataStartOf(file, StreamNumber); + + using var document = PdfDocument.Open(file); + document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required().Data.Length.Should().Be(data.Length); + + var diagnostic = document.Diagnostics.Should().ContainSingle(d => d.Code == PdfDiagnosticCodes.StreamTruncated).Which; + diagnostic.Message.Should().Be(Invariant($"The stream has no endstream: its object's endobj follows the {data.Length} bytes it declared, which are taken as its data.")); + diagnostic.Position.Should().Be(dataStart); + document.Reader.TryGetStreamLengthFault(StreamNumber, out var fault).Should().BeTrue(); + fault.EndStream.Should().Be(EndStreamState.MissingBeforeEndObj); + fault.Taken.Should().Be(data.Length); + fault.Found.Should().BeNull(); + document.Reader.IsEndObjMissing(StreamNumber, out _).Should().BeFalse("the object ends with its endobj"); + document.GetObject(new PdfObjectId(6)).Should().BeOfType(); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged(bool pastWindow) { - // iPRES's t02-05-01-014: the data runs to the end of the file, which the endobj after it does not stop. + // A /Length 20,000 bytes too long, an endobj after the data, then the trailer: no endstream, no object after it, and + // the endobj does not end the file, which is where the search stops. The data runs to the end of the file, and + // holds that endobj, which may be the object's own: whether the object ends with one is not judged (#174). var data = Data(pastWindow ? 3 * Window : 60); var file = Encoding.ASCII.GetBytes( "%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n" + @@ -983,12 +1127,11 @@ public void A_stream_that_endobj_follows_without_its_endstream_says_what_it_take document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required().Data.Length.Should().Be((int)rest); var diagnostic = document.Diagnostics.Should().ContainSingle(d => d.Code == PdfDiagnosticCodes.StreamTruncated).Which; - diagnostic.Message.Should().Be(Invariant($"The stream has no endstream before the endobj that follows its data; the {rest} bytes to the end of the file are taken as its data.")); - diagnostic.Position.Should().Be(dataStart); + diagnostic.Message.Should().Be(Invariant($"The stream declared {data.Length + 20000} bytes, and no endstream follows its data before the end of the file; the {rest} bytes to the end of the file are taken as its data.")); document.Reader.TryGetStreamLengthFault(StreamNumber, out var fault).Should().BeTrue(); - fault.EndStream.Should().Be(EndStreamState.MissingBeforeEndObj); + fault.EndStream.Should().Be(EndStreamState.MissingBeforeEndOfFile); fault.Taken.Should().Be((int)rest); - fault.Found.Should().BeNull(); + document.Reader.IsEndObjMissing(StreamNumber, out _).Should().BeFalse("the data holds an endobj, which may be the object's own"); } [Fact] @@ -1003,7 +1146,7 @@ public void A_stream_the_end_of_the_file_cuts_records_what_it_took() document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required().Data.Length.Should().Be(data.Length); document.Diagnostics.Should().ContainSingle(d => d.Code == PdfDiagnosticCodes.StreamTruncated) - .Which.Message.Should().Be("A stream ran past the end of the file."); + .Which.Message.Should().Be("The stream declared 20000 bytes, and no endstream follows its data before the end of the file; the 200 bytes to the end of the file are taken as its data."); document.Reader.TryGetStreamLengthFault(StreamNumber, out var fault).Should().BeTrue(); fault.Should().Be(new StreamLengthFault { @@ -1016,17 +1159,40 @@ public void A_stream_the_end_of_the_file_cuts_records_what_it_took() } [Fact] - public void A_stream_a_limit_cuts_records_nothing_of_its_length() + public void A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s() { - // The window stopped at the guard, not at the end of the file: what the parser found there is the limit's. + // A stream with no /Length, its data three windows long, under a MaxObjectLength of one window: the window grew to + // find its endstream, reached the guard, and the guard was reported in place of the stream's fault. Stream data is + // searched past the window now, and counts for nothing toward the object's length (#174). var file = Document((StreamNumber, "<< >>\nstream\n" + Data(3 * Window) + "\nendstream")); var options = PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxObjectLength = Window } }; + using var document = PdfDocument.Open(file, options); + document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required().Data.Length.Should().Be(3 * Window); + + document.Diagnostics.Should().ContainSingle().Which.Message.Should().Be( + Invariant($"The stream has no /Length; its data ends after {3 * Window} bytes.")); + document.Reader.TryGetStreamLengthFault(StreamNumber, out var fault).Should().BeTrue(); + fault.EndStream.Should().Be(EndStreamState.Found); + document.Reader.IsCutAtLimit(StreamNumber).Should().BeFalse(); + } + + [Fact] + public void A_stream_whose_data_start_a_limit_cuts_records_nothing_of_its_length() + { + // The window MaxObjectLength allows ends on the stream keyword: where the data starts is past it, the guard's, and + // what the parser found of the stream is the limit's. + var header = Invariant($"{StreamNumber} 0 obj\n"); + var dictionary = "<< /Pad (" + new string('x', 1024 - header.Length - "<< /Pad () >>\nstream".Length) + ") >>\nstream"; + var file = Document((StreamNumber, dictionary + "\n" + Data(100) + "\nendstream")); + var options = PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxObjectLength = 1024 } }; + using var document = PdfDocument.Open(file, options); document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required(); document.Diagnostics.Should().ContainSingle().Which.Code.Should().Be(PdfDiagnosticCodes.LimitObject); document.Reader.TryGetStreamLengthFault(StreamNumber, out _).Should().BeFalse(); + document.Reader.IsCutAtLimit(StreamNumber).Should().BeTrue(); } [Fact] @@ -1064,7 +1230,8 @@ public void The_next_object_a_search_stops_at_is_placed_from_the_header_junk_pre // Six hundred bytes of junk come before %PDF-, and the index counts from there. Declared 300 bytes short, the // stream's endstream lies past where object 6's entry would place it counted from the start of the file: it is // found only when the bound is placed from the header. Without an endstream, the report names where object 6 - // starts in the file, and the search stops there rather than at object 6's endstream. + // starts in the file, and the search stops there rather than at object 6's endstream: object 5's endobj before it + // ends the data (#174). const int Junk = 600; var data = Data(3 * Window); var built = endStreamFollowsData @@ -1081,7 +1248,8 @@ public void The_next_object_a_search_stops_at_is_placed_from_the_header_junk_pre document.Reader.HeaderOffset.Should().Be(Junk); document.Reader.TryGetStreamLengthFault(StreamNumber, out var fault).Should().BeTrue(); - var message = document.Diagnostics.Should().ContainSingle(d => d.Code == PdfDiagnosticCodes.StreamLengthInvalid).Which.Message; + var code = endStreamFollowsData ? PdfDiagnosticCodes.StreamLengthInvalid : PdfDiagnosticCodes.StreamTruncated; + var message = document.Diagnostics.Should().ContainSingle(d => d.Code == code).Which.Message; if (endStreamFollowsData) { @@ -1090,10 +1258,10 @@ public void The_next_object_a_search_stops_at_is_placed_from_the_header_junk_pre } else { - stream.Data.Length.Should().Be(data.Length + 100); + stream.Data.Length.Should().Be(data.Length + 1); fault.NextObject.Should().Be(next); message.Should().Be( - Invariant($"The stream declared {data.Length + 100} bytes, and no endstream follows them before the next object, at {next}; the declared length is kept.")); + Invariant($"The stream declared {data.Length + 100} bytes, and no endstream follows its data before the next object, at {next}: its object's endobj ends it, after {data.Length + 1} bytes.")); } } @@ -1188,7 +1356,8 @@ public void A_stream_past_the_window_holds_the_same_data_whether_it_is_read_befo // Object 5 lost its endstream. Object 7, which the index as written marks free, follows it; object 8's entry // leads nowhere near it, and asking for it rebuilds the index, which places object 7. The index as written places // nothing between object 5 and object 6, but the bytes hold object 7's header: read before the rebuild or after - // it, the search stops there, and the stream keeps its declared length rather than object 7's header and data. The + // it, the search stops there, and the stream ends at its own endobj before it rather than take in object 7's header + // and data (#174). The // rebuilt index is not asked, which would have bounded a stream read after the rebuild only: its data would depend // on what was read first. var data = Data(3 * Window); @@ -1217,13 +1386,13 @@ public void A_stream_past_the_window_holds_the_same_data_whether_it_is_read_befo document.WasRepaired.Should().BeTrue(); var again = document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required(); - first.Data.Length.Should().Be(data.Length + 400); - again.Data.Length.Should().Be(data.Length + 400); + first.Data.Length.Should().Be(data.Length); + again.Data.Length.Should().Be(data.Length); document.GetObject(new PdfObjectId(7)).AsStream().Required().Data.Length.Should().Be(5, "the rebuilt index serves object 7 in its own right"); document.Reader.TryGetStreamLengthFault(StreamNumber, out var fault).Should().BeTrue(); fault.NextObject.Should().Be(seven); - document.Diagnostics.Should().ContainSingle(d => d.Code == PdfDiagnosticCodes.StreamLengthInvalid).Which.Message.Should().Be( - Invariant($"The stream declared {data.Length + 400} bytes, and no endstream follows them before the next object, at {seven}; the declared length is kept.")); + document.Diagnostics.Should().ContainSingle(d => d.Code == PdfDiagnosticCodes.StreamTruncated).Which.Message.Should().Be( + Invariant($"The stream declared {data.Length + 400} bytes, and no endstream follows its data before the next object, at {seven}: its object's endobj ends it, after {data.Length} bytes.")); } [Theory] @@ -1234,7 +1403,8 @@ public void A_stream_past_the_window_stops_where_the_index_as_written_places_the // As the IBM QMF manual's entries do, object 16's points one byte into its header, at "6 0 obj": reading object 16 // finds it a byte before, and corrects the index the reader reads with. The search for object 5's endstream, which // it lost, is bounded by the index as the file wrote it, and stops at the same place whether object 16 was read and - // found first or not. + // found first or not. Its endobj lies before that place, but a byte of object 16's header lies between: the data + // runs to the place the index gives, and holds the endobj, whether the object ends there unknown (#174). var data = Data(3 * Window); var file = Document((StreamNumber, Stream(data.Length + 100, data, end: "\n")), (16, "(" + new string('p', 200) + ")")); var header = HeaderOf(file, 16); @@ -1252,11 +1422,13 @@ public void A_stream_past_the_window_stops_where_the_index_as_written_places_the var stream = document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required(); - stream.Data.Length.Should().Be(data.Length + 100); + var taken = (int)(header + 1 - DataStartOf(file, StreamNumber)); + stream.Data.Length.Should().Be(taken); document.Reader.TryGetStreamLengthFault(StreamNumber, out var fault).Should().BeTrue(); fault.NextObject.Should().Be(header + 1, "the index as written places object 16 there"); document.Diagnostics.Should().ContainSingle(d => d.Code == PdfDiagnosticCodes.StreamLengthInvalid).Which.Message.Should().Be( - Invariant($"The stream declared {data.Length + 100} bytes, and no endstream follows them before the next object, at {header + 1}; the declared length is kept.")); + Invariant($"The stream declared {data.Length + 100} bytes, and no endstream follows its data before the next object, at {header + 1}; the {taken} bytes up to that object are taken as its data.")); + document.Reader.IsEndObjMissing(StreamNumber, out _).Should().BeFalse("the data holds an endobj, which may be the object's own"); } [Theory] @@ -1296,10 +1468,10 @@ public void An_index_rebuilt_as_the_document_opened_bounds_the_searches_as_it_st document.GetObject(new PdfObjectId(number)); } - document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required().Data.Length.Should().Be(data.Length + 100); + document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required().Data.Length.Should().Be(data.Length + 1); document.GetObject(new PdfObjectId(7)).AsStream().Required().Data.Length.Should().Be(5); - document.Diagnostics.Should().ContainSingle(d => d.Code == PdfDiagnosticCodes.StreamLengthInvalid).Which.Message.Should().Be( - Invariant($"The stream declared {data.Length + 100} bytes, and no endstream follows them before the next object, at {HeaderOf(file, 6)}; the declared length is kept.")); + document.Diagnostics.Should().ContainSingle(d => d.Code == PdfDiagnosticCodes.StreamTruncated).Which.Message.Should().Be( + Invariant($"The stream declared {data.Length + 100} bytes, and no endstream follows its data before the next object, at {HeaderOf(file, 6)}: its object's endobj ends it, after {data.Length + 1} bytes.")); source.Searches(dataStart, data.Length).Should().Be(0, "the stream was searched as the document opened"); document.Reader.Index.Entries.Should().Equal(opened, "nothing read after the document opened changes an index rebuilt as it opened"); } @@ -1315,7 +1487,8 @@ public void The_next_object_a_search_stops_at_is_where_the_index_as_written_plac // its endstream (#182). Once the chain is read, the /Length is read and checked: object 10 is found near its entry, // which is corrected behind a copy of the index as the file wrote it. Object 5 lost its endstream; its search stops // where that index places the next object, 2 bytes before object 10's header, whether reading object 6 — whose entry - // points one byte into its header — corrected the index the reader reads with since, or not. + // points one byte into its header — corrected the index the reader reads with since, or not. That place cuts object + // 5's endobj: the data runs to it (#174). var data = Data(3 * Window); var file = LengthObjectOffItsEntry(data, out var tenAt); @@ -1332,12 +1505,13 @@ public void The_next_object_a_search_stops_at_is_where_the_index_as_written_plac var stream = document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required(); - stream.Data.Length.Should().Be(data.Length + 100); + var taken = (int)(tenAt - 2 - DataStartOf(file, StreamNumber)); + stream.Data.Length.Should().Be(taken); document.Reader.TryGetStreamLengthFault(StreamNumber, out var fault).Should().BeTrue(); fault.NextObject.Should().Be(tenAt - 2); document.Diagnostics.Should().ContainSingle(d => d.Code == PdfDiagnosticCodes.StreamLengthInvalid && d.Position == DataStartOf(file, StreamNumber)) .Which.Message.Should().Be( - Invariant($"The stream declared {data.Length + 100} bytes, and no endstream follows them before the next object, at {tenAt - 2}; the declared length is kept.")); + Invariant($"The stream declared {data.Length + 100} bytes, and no endstream follows its data before the next object, at {tenAt - 2}; the {taken} bytes up to that object are taken as its data.")); } [Theory] @@ -1594,6 +1768,26 @@ public void A_remote_document_reads_the_stream_whose_length_is_wrong_as_the_meas .Which.Message.Should().Be(message); } + [Fact] + public void IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows() + { + // t02-05-01-014: the 68 bytes its /Length declares, which its endobj follows without an endstream. It took the 310 + // bytes to the end of the file, through object 5, the table and the trailer (#174). + const string File = "remote/ipres2017/t02-05-01-014-endstream-keyword-missing.pdf"; + Assert.SkipUnless(Corpus.Paths.Contains(File), $"{File} is a remote document this machine did not fetch"); + using var document = PdfDocument.Open(Corpus.Read(File), CorpusOptions(Corpus.Get(File))); + + foreach (var each in document.ObjectNumbers.ToList()) + { + document.GetObject(new PdfObjectId(each)); + } + + document.Reader.TryGetStreamLengthFault(4, out var fault).Should().BeTrue(); + fault.Taken.Should().Be(68); + document.Diagnostics.Should().ContainSingle(d => d.Position == fault.DataStart).Which.ToString().Should().Be( + "Warning stream.truncated at 315: The stream has no endstream: its object's endobj follows the 68 bytes it declared, which are taken as its data."); + } + private static readonly (string File, int Number, long DataStart, int Length, string Message)[] RemoteRows = [ ("remote/opf-format-corpus/ibm-id-workbench-xpp-qmf-manual.pdf", 93, 89018, 115264, "The stream declared 115266 bytes but ended after 115264."), @@ -1609,6 +1803,7 @@ private static readonly (string File, int Number, long DataStart, int Length, st ("remote/opf-format-corpus/quartz-word-samhsa-prevention-pathways-fact-sheet.pdf", 72, 145068, 8458, "The stream declared 8434 bytes but ended after 8458."), ("remote/opf-format-corpus/jhove-hul-35-atypon-pdfplus-journal-article.pdf", 49, 249330, 1647, "The stream declared 62065 bytes but ended after 1647."), ("remote/opf-format-corpus/jhove-hul-35-pdfium-urban-planning-report.pdf", 695, 19901349, 202154, "The stream declared 202154 bytes, and no endstream follows them before the next object, at 20103524; the declared length is kept."), + ("remote/pdfbox/distiller6-zeroed-object-stream-pdfbox3947.pdf", 417, 1670741, 1302, "The stream declared 1302 bytes, and no endstream follows them before the next object, at 1672062; the declared length is kept."), ("remote/ipres2017/t02-05-01-015-invalid-stream-length-in-dictionary.pdf", 4, 316, 68, "The stream's /Length is a real number, 61.5, not a non-negative integer; its data ends after 68 bytes."), ("remote/ipres2017/t02-05-01-016-stream-length-missing-in-dictionary.pdf", 4, 304, 68, "The stream has no /Length; its data ends after 68 bytes."), ("remote/opf-format-corpus/jhove-hul-157-tiff2pdf-1996-journal-article-scan.pdf", 16, 542577, 5555, "The stream's /Length names object 18 0, which holds a stream, not a non-negative integer; its data ends after 5555 bytes."), diff --git a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs index 69f2b3b7..22055320 100644 --- a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs +++ b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs @@ -110,7 +110,7 @@ public void A_stream_ending_just_before_the_window_edge_is_confirmed_without_a_l { // T21's shape: the data ends 0 to 10 bytes short of the window's edge — the hospital-bed guidance's // objects 604 and 2053 end 2 bytes short and right on it, before a CR LF — and its "endstream" lies - // across or past the edge. The reader asks the file for the 13 bytes after the data instead of + // across or past the edge. The reader asks the file for a few bytes after the data instead of // parsing the object again through a window eight times larger, which the file, padded past that // size, would otherwise serve. var failures = new List(); @@ -132,7 +132,7 @@ public void A_stream_ending_just_before_the_window_edge_is_confirmed_without_a_l var read = source.BytesRead - beforeLoading; var length = stream.GetRawBytes().Length; - if (read > Window + 13 || length != Window - shortOfEdge - DataStart(Window - shortOfEdge) || + if (read > Window + PdfObjectParser.EndObjLookahead || length != Window - shortOfEdge - DataStart(Window - shortOfEdge) || document.Diagnostics.Count > 0) { failures.Add($"{shortOfEdge} bytes short: read {read} bytes, data of {length}, {document.Diagnostics.Count} diagnostics"); @@ -376,10 +376,11 @@ public void A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream( } [Fact] - public void A_stream_the_file_cuts_short_is_reported_without_asking_the_source_past_its_end() + public void A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end() { // The file ends inside "endstream". The reader asks the file for the bytes after the data, and a - // source may refuse a read that runs past its end: it must be asked only for the six there are. + // source may refuse a read that runs past its end: it must be asked only for the six there are. No endstream + // nor endobj follows the declared length, which ends before the end of the file: it is kept, and reported (#174). var bytes = Encoding.ASCII.GetBytes( "%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n" + "2 0 obj << /Type /Pages /Kids [] /Count 0 >> endobj\n" + @@ -388,8 +389,9 @@ public void A_stream_the_file_cuts_short_is_reported_without_asking_the_source_p using var document = PdfDocument.Open(new StrictCountingSource(bytes), options: null, ownsSource: true); var stream = document.GetObject(new PdfObjectId(3)).AsStream().Required(); - Encoding.ASCII.GetString(stream.GetRawBytes().Span).Should().StartWith("hello"); - document.Diagnostics.Contains(PdfDiagnosticCodes.StreamTruncated).Should().BeTrue(); + Encoding.ASCII.GetString(stream.GetRawBytes().Span).Should().Be("hello"); + document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.StreamLengthInvalid).Which.Message.Should().Be( + "The stream declared 5 bytes, and no endstream follows them before the end of the file; the declared length is kept."); } /// From f8f83dab452698fa4a9dffc21ec7966657e12449 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 06:07:28 +0000 Subject: [PATCH 004/146] fix: end an object's open containers at the next object's header An object that lost both its closing delimiter and its endobj had its array or dictionary take in the next object's header and value, up to that object's endobj, and read as ending with one: the missing endobj went unreported, and a trailer that lost its >> took the next update's entries. Inside an array or a dictionary, where an element, a key or a value is expected, the next object's header now ends every container left open. A header is the one the search for a stream's endstream stops at: N G obj, digits alone, N from 1 to the largest object number and G within the generation range, white space alone between them, and obj a whole keyword a window's edge does not reach. The cut is reported once, at the innermost container, as "An object header ended the object", and the object lacks its endobj. Containers skipped past the depth the parser follows end there too. An object stream's member is never followed by a header, and a value alone is no container: neither applies the rule. Its cost: a closed container holding text that reads as a header ends there, and its endobj is judged missing. None of the 344,361 objects of the corpus holds one. Refs #230 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 139 ++++++++++++++++-- tests/AdCodicem.Pdf.Tests/ParserTests.cs | 54 +++++++ tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs | 138 +++++++++++++++++ tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs | 29 ++++ 4 files changed, 350 insertions(+), 10 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 22c9d91f..e9f80bb3 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -62,6 +62,12 @@ internal ref struct PdfObjectParser /// private bool _cutReported; + /// + /// Where the header of the object after this one starts, in the buffer, when one ended the containers this object left + /// open, where a value or a key was expected (#230); -1 when none did. + /// + private int _headerAt = -1; + /// /// Whether the last stream read kept a declared length that neither endstream nor endobj follows: where /// its object ends is unknown, and its endobj is not judged. @@ -189,7 +195,11 @@ public int Position } /// Parses the object at the current position. - public PdfObject ParseObject() => ParseValue(_lexer.Read(), 0); + public PdfObject ParseObject() + { + _headerAt = -1; + return ParseValue(_lexer.Read(), 0); + } /// /// Determines whether an object header may still begin with and , @@ -252,6 +262,7 @@ public bool TryReadIndirectObject(out PdfObjectId id, out PdfObject value) id = new PdfObjectId((int)number.Integer, (int)generation.Integer); _objectNumber = id.Number; _headerStart = number.Start; + _headerAt = -1; // An empty object, "2 0 obj endobj", reads its endobj as its value, and says so; that endobj is its own. var beforeValue = _lexer.Position; @@ -325,7 +336,7 @@ private PdfObject ParseValue(PdfToken token, int depth) return PdfNull.Instance; case PdfTokenKind.Integer: - return ParseIntegerOrReference(token); + return ParseIntegerOrReference(token, depth); // A token a window's edge cut is the guard's, or the next window's, to judge: nothing is reported of it. case PdfTokenKind.Real when !double.IsFinite(token.Real) && AtWindowEdge(token): @@ -383,7 +394,11 @@ private PdfObject ParseValue(PdfToken token, int depth) } } - private PdfObject ParseIntegerOrReference(PdfToken token) + /// + /// Parses an integer, or the reference it starts, inside containers: inside one, the header of + /// the next object it starts ends them (#230), and reads as null where says. + /// + private PdfObject ParseIntegerOrReference(PdfToken token, int depth) { // "12 0 R" is only distinguishable from "12" by looking two tokens ahead. "0 0 R" is a reference too: object 0 // heads the free list and is never in use (ISO 32000-1, 7.5.4), so it reads as null, as a reference to any @@ -399,6 +414,15 @@ private PdfObject ParseIntegerOrReference(PdfToken token) return new PdfReference(new PdfObjectId((int)token.Integer, (int)second.Integer), _source); } + // The next object's header where an element or a value should be: this object lost both its closing delimiter and + // its endobj, and its containers end where the next object starts (#230). A value alone is no container. + if (depth > 0 && IsHeader(token, second, third)) + { + _headerAt = token.Start; + _lexer.Position = saved; + return PdfNull.Instance; + } + // The buffer ended before the look-ahead could tell: "12 0" may be all there is, or what a // window's edge left of "12 0 R". if (third.Kind == PdfTokenKind.EndOfInput) @@ -447,14 +471,34 @@ private PdfObject ParseArray(int depth, int openedAt) return array; // An endobj where an element should be ends the object, the array with it, as PDFBox and pdfium read it: - // the array takes nothing past it, though it is the next object's when this one lost its own (#230). + // the array takes nothing past it. An object that lost its own endobj meets the next object's header + // first, which ends it there (#230). case PdfTokenKind.Keyword when IsEndObj(token): _lexer.Position = token.Start; ReportCut("an array", openedAt, depth, atEndObj: true); return array; default: - array.Add(ParseValue(token, depth + 1)); + var element = ParseValue(token, depth + 1); + + if (_headerAt >= 0) + { + // The next object's header ends the array: where an element should be, reported here; inside an + // array or a dictionary the element opened, reported there, and that element kept as it was read. + if (_headerAt == token.Start) + { + ReportCut("an array", openedAt, depth, atHeader: true); + } + else + { + array.Add(element); + } + + _lexer.Position = _headerAt; + return array; + } + + array.Add(element); if (IsMemberStream) { @@ -498,8 +542,8 @@ private PdfObject ParseDictionaryOrStream(int depth, int openedAt) return dictionary; } - // An endobj where a key should be ends the object, the dictionary with it: the dictionary takes nothing past it, - // though it is the next object's when this one lost its own (#230). + // An endobj where a key should be ends the object, the dictionary with it: the dictionary takes nothing past it. + // An object that lost its own endobj meets the next object's header first, which ends it there (#230). if (IsEndObj(keyToken)) { _lexer.Position = keyToken.Start; @@ -509,6 +553,14 @@ private PdfObject ParseDictionaryOrStream(int depth, int openedAt) if (keyToken.Kind != PdfTokenKind.Name) { + if (StartsHeader(keyToken)) + { + _headerAt = keyToken.Start; + _lexer.Position = keyToken.Start; + ReportCut("a dictionary", openedAt, depth, atHeader: true); + return dictionary; + } + // A string where a key should be is a string all the same, which the end of the data can leave open inside // the dictionary: it is the innermost construct then. if (keyToken.Kind is PdfTokenKind.LiteralString) @@ -566,6 +618,24 @@ private PdfObject ParseDictionaryOrStream(int depth, int openedAt) return dictionary; } + if (_headerAt >= 0) + { + // The next object's header ends the dictionary: where the value should be, the key given none and reported + // here; inside an array or a dictionary the value opened, reported there, and that value kept. + if (_headerAt == valueToken.Start) + { + ReportCut("a dictionary", openedAt, depth, valueMissing: true, atHeader: true); + Give(dictionary, ref nullKeys, keyToken, key, PdfNull.Instance, keyAtEdge, valueCut: false); + } + else + { + Give(dictionary, ref nullKeys, keyToken, key, value, keyAtEdge, valueCut: false); + } + + _lexer.Position = _headerAt; + return dictionary; + } + // A value the parse met a window's edge in may be what the edge made of it — a keyword cut short reads as null —: // it is kept as far as it was read, but removes nothing. Give(dictionary, ref nullKeys, keyToken, key, value, keyAtEdge, valueCut: PastWindowEdge); @@ -1069,6 +1139,12 @@ private void SkipContainer(PdfTokenKind endKind, int openedAt, int enclosing) ReportCut(endKind == PdfTokenKind.ArrayEnd ? "an array" : "a dictionary", openedAt, enclosing, atEndObj: true); return; + case PdfTokenKind.Integer when StartsHeader(token): + _headerAt = token.Start; + _lexer.Position = token.Start; + ReportCut(endKind == PdfTokenKind.ArrayEnd ? "an array" : "a dictionary", openedAt, enclosing, atHeader: true); + return; + case PdfTokenKind.ArrayStart when endKind == PdfTokenKind.ArrayEnd: case PdfTokenKind.DictionaryStart when endKind == PdfTokenKind.DictionaryEnd: depth++; @@ -1227,9 +1303,11 @@ private void ReportIfUnterminated(PdfToken token, string what, int depth) /// The bytes a string took to the end of the data, or 0 for an array or a dictionary. /// Whether the cut came after a dictionary's key, before its value. /// Whether an endobj made the cut, rather than the end of the data. - private void ReportCut(string what, int openedAt, int enclosing, int swallowed = 0, bool valueMissing = false, bool atEndObj = false) + /// Whether the next object's header made the cut, rather than the end of the data (#230). + private void ReportCut( + string what, int openedAt, int enclosing, int swallowed = 0, bool valueMissing = false, bool atEndObj = false, bool atHeader = false) { - if ((!_endsData && !atEndObj) || _cutReported) + if ((!_endsData && !atEndObj && !atHeader) || _cutReported) { return; } @@ -1242,7 +1320,7 @@ private void ReportCut(string what, int openedAt, int enclosing, int swallowed = return; } - var data = atEndObj ? "An endobj ended the object" : End; + var data = atEndObj ? "An endobj ended the object" : atHeader ? "An object header ended the object" : End; var content = swallowed > 0 ? string.Create(CultureInfo.InvariantCulture, $", which takes the {swallowed:N0} {(swallowed == 1 ? "byte" : "bytes")} from where it opens to that end") : valueMissing ? ", which was never closed, before the value of its last key" : ", which was never closed"; @@ -1276,6 +1354,47 @@ private void ReportCut(string what, int openedAt, int enclosing, int swallowed = /// private readonly bool IsEndObj(PdfToken token) => token.IsKeyword("endobj"u8) && !AtWindowEdge(token); + /// + /// Determines whether , read where a key is expected, starts the next object's header + /// (): the two tokens after it are read, and the lexer left where it was. + /// + private bool StartsHeader(PdfToken number) + { + if (number.Kind != PdfTokenKind.Integer || _member is not null) + { + return false; + } + + var saved = _lexer.Position; + var generation = _lexer.Read(); + var keyword = generation.Kind == PdfTokenKind.Integer ? _lexer.Read() : generation; + _lexer.Position = saved; + return IsHeader(number, generation, keyword); + } + + /// + /// Determines whether three tokens are an object header as the bytes hold one, the rule the search for a stream's + /// endstream stops at (): N G obj, digits alone for a number from 1 to + /// 2,147,483,647 and a generation from 0 to 65,535, white space alone between them — no comment —, and obj a + /// whole keyword, which one a window's edge reaches may not be. An object stream's member is never followed by a + /// header: its members have none (#230). + /// + private readonly bool IsHeader(PdfToken number, PdfToken generation, PdfToken keyword) + { + if (_member is not null || number.Kind != PdfTokenKind.Integer || generation.Kind != PdfTokenKind.Integer || + !keyword.IsKeyword("obj"u8) || AtWindowEdge(keyword) || + number.Integer is < 1 or > PdfObjectId.MaxNumber || generation.Integer is < 0 or > PdfObjectId.MaxGeneration) + { + return false; + } + + var span = _memory.Span; + return !span[number.Start..number.End].ContainsAnyExceptInRange((byte)'0', (byte)'9') && + !span[generation.Start..generation.End].ContainsAnyExceptInRange((byte)'0', (byte)'9') && + !span[number.End..generation.Start].ContainsAnyExcept(PdfCharacters.Whitespace) && + !span[generation.End..keyword.Start].ContainsAnyExcept(PdfCharacters.Whitespace); + } + /// Gets what ended the buffer, as a report of what it cut short names it. private readonly string End => _member switch { diff --git a/tests/AdCodicem.Pdf.Tests/ParserTests.cs b/tests/AdCodicem.Pdf.Tests/ParserTests.cs index e48facfd..abc96759 100644 --- a/tests/AdCodicem.Pdf.Tests/ParserTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ParserTests.cs @@ -163,6 +163,60 @@ public void Asks_for_more_data_only_for_a_header_its_end_cuts_that_no_whole_toke parser.IsTruncated.Should().Be(more); } + [Fact] + public void Ends_the_containers_an_object_left_open_at_the_next_object_s_header_and_leaves_it_to_be_read() + { + // The object lost its ] and its endobj: the next object's header ends the array, and is where the parser stands to + // read that object (#230). + var diagnostics = new PdfDiagnostics(); + var parser = new PdfObjectParser(Encoding.ASCII.GetBytes("[1 2\n6 0 obj\n(six)\nendobj\n"), diagnostics: diagnostics); + + parser.ParseObject().AsArray().Required().Should().HaveCount(2); + parser.TryReadIndirectObject(out var id, out var value).Should().BeTrue(); + + id.Should().Be(new PdfObjectId(6)); + value.Should().BeOfType(); + diagnostics.Should().ContainSingle().Which.Message.Should().Be("An object header ended the object inside an array, which was never closed."); + } + + [Theory] + [InlineData("[1 2\n6 0 obj", true, 5)] + [InlineData("[1 2\n6 0 obj ", false, 2)] + [InlineData("[1 2\n6 0 obj\n", false, 2)] + public void Takes_an_object_header_a_window_s_edge_reaches_for_none_until_a_larger_window_shows_obj_whole( + string text, bool more, int count) + { + // "obj" the edge reaches may be the start of "objx", no keyword at all: what the window reads is dropped for a larger + // window, which says (#230). + var parser = new PdfObjectParser(Encoding.ASCII.GetBytes(text), endsData: false); + + parser.ParseObject().AsArray().Required().Should().HaveCount(count); + parser.IsTruncated.Should().Be(more); + } + + [Theory] + [InlineData("<< /A [1 << /B 2\n6 0 obj\n", 2)] + [InlineData("<< /A 1\n6 0 obj\n", 1)] + public void A_header_where_a_key_or_a_value_is_expected_ends_the_dictionary(string text, int depthOfCut) + { + var diagnostics = new PdfDiagnostics(); + var parser = new PdfObjectParser(Encoding.ASCII.GetBytes(text), diagnostics: diagnostics); + + parser.ParseObject().Should().BeOfType(); + + diagnostics.Should().ContainSingle().Which.Position.Should().Be(depthOfCut == 1 ? 0 : text.IndexOf("<< /B", StringComparison.Ordinal)); + parser.IsTruncated.Should().BeFalse(); + } + + [Fact] + public void A_value_alone_that_is_a_header_s_number_is_an_integer() + { + // Outside a container a header ends nothing: "4 0 obj 5 0 obj" holds the integer 5, and its endobj is missing (#230). + var parser = new PdfObjectParser(Encoding.ASCII.GetBytes("5 0 obj\n")); + + parser.ParseObject().Should().BeOfType().Which.Value.Should().Be(5); + } + [Fact] public void Reads_a_stream_whose_declared_length_is_correct() { diff --git a/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs b/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs index 16e98f02..8b903ac0 100644 --- a/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs +++ b/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs @@ -144,6 +144,144 @@ public void A_container_an_endobj_ends_takes_nothing_of_the_objects_after_it(str new PdfValidator().Validate(document).Findings.Should().BeEmpty(); } + [Theory] + [InlineData("[1 2 3", "an array, which was never closed")] + [InlineData("<< /A 1", "a dictionary, which was never closed")] + [InlineData("<< /A", "a dictionary, which was never closed, before the value of its last key")] + public void An_object_that_lost_its_closing_delimiter_and_its_endobj_ends_at_the_next_object_s_header(string value, string what) + { + // Object 4 lost both its closing ] or >> and its endobj: its container took in object 5's header and value up to + // object 5's endobj, and the object read as ending with one, its missing endobj unreported (#230). The next object's + // header ends it, the cut reported once, and the object lacks its endobj. + var file = FourLostItsEnd(value); + using var document = PdfDocument.Open(file); + + var read = document.GetObject(new PdfObjectId(4)); + + (read is PdfArray { Count: 3 } || read is PdfDictionary { Count: <= 1 }).Should().BeTrue(); + (read is not PdfDictionary dictionary || dictionary.ContainsKey(PdfName.Get("A")) == !value.EndsWith("/A", StringComparison.Ordinal)).Should().BeTrue(); + document.GetObject(new PdfObjectId(5)).AsDictionary().Required().Should().HaveCount(2); + var report = document.Diagnostics.Should().ContainSingle().Subject; + report.Code.Should().Be(PdfDiagnosticCodes.SyntaxTruncatedObject); + report.Position.Should().Be(PdfTemplate.OffsetOf(file, "4 0 obj\n") + "4 0 obj\n".Length); + report.Message.Should().Be($"An object header ended the object inside {what}."); + document.Reader.IsEndObjMissing(4, out _).Should().BeTrue(); + new PdfValidator().Validate(document).Findings.Should().ContainSingle() + .Which.RuleId.Should().Be(PdfValidationRuleIds.ObjectEndObjMissing); + } + + [Fact] + public void The_next_object_s_header_ends_every_container_left_open_and_is_reported_at_the_innermost() + { + // The innermost dictionary took object 5's /Foo and /Author and closed at its >>: it ends at the header, which is + // reported where it opens, and the containers around it end there too (#230). + var file = FourLostItsEnd("<< /A [1 << /B 2"); + using var document = PdfDocument.Open(file); + + var outer = document.GetObject(new PdfObjectId(4)).AsDictionary().Required(); + var array = outer.GetArray(PdfName.Get("A")).Required(); + array.Should().HaveCount(2); + array[1].Should().BeOfType().Which.Should().HaveCount(1, "the inner dictionary takes nothing of object 5"); + var report = document.Diagnostics.Should().ContainSingle().Subject; + report.Position.Should().Be(PdfTemplate.OffsetOf(file, "<< /B 2")); + report.Message.Should().Be("An object header ended the object inside a dictionary, which was never closed; the 2 arrays or dictionaries around it were never closed either."); + document.Reader.IsEndObjMissing(4, out _).Should().BeTrue(); + } + + [Theory] + [InlineData("[1 (x 6 0 obj y) 2", 3, 0)] + [InlineData("[1 6 0 R", 2, 0)] + [InlineData("[1 6 65536 obj", 4, 1)] + [InlineData("[1 2147483648 0 obj", 4, 1)] + [InlineData("[1 6 %a comment\n0 obj", 4, 1)] + [InlineData("[1 +6 0 obj", 4, 1)] + public void Text_that_is_no_object_header_ends_no_container(string value, int count, int strays) + { + // A header in a string is no token; a reference is decided first; a generation or a number out of range, a comment + // between the tokens, a sign before the number: none is a header, as the search for a stream's endstream takes + // one, and the array goes on, a stray obj reported, to object 5's header (#230). + var file = FourLostItsEnd(value); + using var document = PdfDocument.Open(file); + + document.GetObject(new PdfObjectId(4)).AsArray().Required().Should().HaveCount(count); + document.Diagnostics.Count(entry => entry.Code == PdfDiagnosticCodes.SyntaxUnexpectedToken).Should().Be(strays); + document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.SyntaxTruncatedObject) + .Which.Message.Should().Be("An object header ended the object inside an array, which was never closed."); + } + + [Fact] + public void A_header_inside_a_closed_container_ends_it_there_and_its_endobj_is_judged_missing() + { + // The rule's cost: text an object holds that is a header ends the object there, though its container closes and + // its endobj follows; what follows the header is not read (#230). No document of the corpus holds one. + var file = PdfTemplate.Build(PdfTemplate.Sound + .Replace("<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Catalog /Pages 2 0 R /Test 4 0 R >>", StringComparison.Ordinal) + .Replace("xref\n0 4\n", "4 0 obj\n[1 2 6 0 obj 3]\nendobj\nxref\n0 5\n", StringComparison.Ordinal) + .Replace("{row:3}\n", "{row:3}\n{row:4}\n", StringComparison.Ordinal) + .Replace("/Size 4", "/Size 5", StringComparison.Ordinal)); + using var document = PdfDocument.Open(file); + + document.GetObject(new PdfObjectId(4)).AsArray().Required().Should().HaveCount(2); + document.Diagnostics.Should().ContainSingle().Which.Message.Should().Be("An object header ended the object inside an array, which was never closed."); + document.Reader.IsEndObjMissing(4, out _).Should().BeTrue(); + } + + [Fact] + public void An_object_stream_member_holding_a_header_reads_it_as_tokens() + { + // A member has no header, and is followed by none: text in it that reads as one is a stray obj, as it was (#230). + var file = Packed("[1 2 5 0 obj 3]", last: true); + using var document = PdfDocument.Open(file); + + document.GetObject(new PdfObjectId(4)).AsArray().Required().Should().HaveCount(6); + document.Diagnostics.Should().ContainSingle().Which.Code.Should().Be(PdfDiagnosticCodes.SyntaxUnexpectedToken); + } + + [Fact] + public void Nesting_deeper_than_the_reader_follows_ends_at_the_next_object_s_header_too() + { + // Past the depth the parser follows, the containers are skipped to their end: an endobj, and now the next object's + // header, ends them (#230). + var file = FourLostItsEnd(new string('[', 200)); + using var document = PdfDocument.Open(file); + + document.GetObject(new PdfObjectId(4)).Should().BeOfType(); + document.GetObject(new PdfObjectId(5)).AsDictionary().Required().Should().HaveCount(2); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.SyntaxDepthExceeded, PdfDiagnosticCodes.SyntaxTruncatedObject); + document.Reader.IsEndObjMissing(4, out _).Should().BeTrue(); + } + + [Fact] + public void A_trailer_that_lost_its_closing_delimiter_takes_nothing_of_the_update_after_it() + { + // An older trailer that lost its >>: it took the update's object 4, /Type and /Info, into the trailer the reader + // merged, and /Info named an object the file lacks. The update's header ends it (#230). + var file = TestPdfBuilder.AppendIncrementalUpdate( + new TestPdfBuilder().WithObject(1, "<< /Type /Catalog /Pages 2 0 R >>").WithObject(2, "<< /Type /Pages /Kids [] /Count 0 >>").BuildClassic(rootNumber: 1), + rootNumber: 1, + [(4, "<< /Type /Foo /Info 9 0 R >>")]); + var text = Encoding.Latin1.GetString(file); + var trailer = text.IndexOf("/Root 1 0 R >>", StringComparison.Ordinal) + "/Root 1 0 R".Length; + Encoding.ASCII.GetBytes(" ").CopyTo(file, trailer); + + using var document = PdfDocument.Open(file); + + document.Trailer.ContainsKey(PdfName.Info).Should().BeFalse(); + document.Trailer.ContainsKey(PdfName.Type).Should().BeFalse(); + new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.ObjectReferenceMissing).Should().BeFalse(); + } + + /// + /// A sound document whose catalog names objects 4 and 5, object 4 holding and no endobj, then + /// object 5 holding and its endobj. + /// + private static byte[] FourLostItsEnd(string value, string five = "<< /Foo 7 /Author (a) >>") => + PdfTemplate.Build(PdfTemplate.Sound + .Replace("<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Catalog /Pages 2 0 R /Test 4 0 R /Other 5 0 R >>", StringComparison.Ordinal) + .Replace("xref\n0 4\n", $"4 0 obj\n{value}\n5 0 obj\n{five}\nendobj\nxref\n0 6\n", StringComparison.Ordinal) + .Replace("{row:3}\n", "{row:3}\n{row:4}\n{row:5}\n", StringComparison.Ordinal) + .Replace("/Size 4", "/Size 6", StringComparison.Ordinal)); + [Theory] [InlineData("/ID [<41zz42> <4142>]", PdfDiagnosticCodes.SyntaxHexStringInvalid)] [InlineData("/Info#zz 3 0 R", PdfDiagnosticCodes.SyntaxNameEscapeInvalid)] diff --git a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs index 22055320..b327b1b8 100644 --- a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs +++ b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs @@ -312,6 +312,35 @@ public void A_cross_reference_stream_after_more_white_space_than_the_object_wind document.Diagnostics.Should().BeEmpty(); } + [Fact] + public void An_object_that_lost_its_end_ends_at_the_next_header_wherever_the_window_edge_cuts_it() + { + // The header that ends an array which lost its ] and its endobj meets the edge at every byte, and a byte or two + // before and after: the edge never makes the header an integer and a stray keyword, nor hides it (#230). + var failures = new List(); + const string Before = "[("; + const string After = ") 1 2 3\n6 0 obj\n(six)"; + var header = Header(ObjectNumber); + + for (var shortOfEdge = -2; shortOfEdge <= 10; shortOfEdge++) + { + var filler = Window - header.Length - Before.Length - ") 1 2 3\n".Length - shortOfEdge; + using var document = PdfDocument.Open(Document(Before + new string('x', filler) + After)); + + var read = document.GetObject(new PdfObjectId(ObjectNumber)); + var reports = string.Join(" | ", document.Diagnostics.Select(entry => entry.Message)); + + if (read is not PdfArray { Count: 4 } || + reports != "An object header ended the object inside an array, which was never closed." || + !document.Reader.IsEndObjMissing(ObjectNumber, out _)) + { + failures.Add($"the header {shortOfEdge} bytes before the edge: {read}, {reports}"); + } + } + + failures.Should().BeEmpty(); + } + [Fact] public void An_object_whose_header_lies_past_the_window_is_read_where_the_index_says() { From 4b7c99299c946d33bffdd274b5e3d36a84973627 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 06:32:16 +0000 Subject: [PATCH 005/146] fix: settle a cross-reference stream's deferred length before its rows are read A cross-reference stream whose /Length the chain cannot read takes its data up to the first endstream. Rows that spell one cut the data short: the section was read in part and called malformed, and an update's objects past the spelled rows read as the revision it supersedes. Data with no endstream at all ran to the end of the file and was read as rows. Where the section's own row for the /Length lies among the rows the data holds, the object it places is read there, nothing kept or reported, and the data is taken again to the length it gives when an endstream confirms it, then decoded once more. A row that does not stand leaves the data as the chain took it, and the length is checked once the chain is read, as before. Data with no endstream before the next object the sections read so far place, or the end of the file, makes its section malformed: the file's fault, and no guard's. The search past the window this relies on came with the single search for a stream's end: a stream of 70,000 bytes of rows reads as its direct twin, with no limit.trailer, and MaxTrailerLength bounds the dictionary alone. Tests hold that too. Refs #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../Documents/PdfReaderLimits.cs | 4 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 166 ++++++++++--- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 198 +++++++++++++-- .../DocumentReaderTests.cs | 228 +++++++++++++++++- .../AdCodicem.Pdf.Tests/HostileInputTests.cs | 19 ++ .../AdCodicem.Pdf.Tests/ReaderLimitsTests.cs | 15 ++ 6 files changed, 575 insertions(+), 55 deletions(-) diff --git a/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs b/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs index 204ac04c..2b44454d 100644 --- a/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs +++ b/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs @@ -134,7 +134,9 @@ public int MaxXRefSectionCount /// /// /// A classic trailer that ends within the window its cross-reference table was read through is read whole, - /// whatever its length: the guard bounds how far the reader follows one past that window. A value above + /// whatever its length: the guard bounds how far the reader follows one past that window. A cross-reference stream's + /// data is not a trailer: one whose /Length the reader cannot read while it reads the chain is searched for its + /// endstream in the file, up to the next object or the end of the file. A value above /// is taken as that length. Reaching it reports /// , and ends the cross-reference chain at the section cut: an /// older section, or a key, past the cut is unknown, and what only older sections index reads as null until the diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 3feb8477..d140394b 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -2099,8 +2099,10 @@ private void Initialize() /// The /Length is loaded as any object is: what that load finds or changes comes after the chain's index is /// copied (). A length the endstream the chain stopped at confirms — the white /// space before it counted, or not all of it — is the stream's own, as the parser takes it for any stream, and says - /// nothing. One confirmed by an endstream past it gives data the chain did not read, rows included: they are not - /// read again, and the report says so. The stream is not parsed again, and each is reported once. + /// nothing; so is one the data was taken again to, the section's own row placing it (). + /// One confirmed by an endstream past it gives data the chain did not read, rows included — the section's row + /// for the /Length among them, or one that does not stand —: they are not read again, and the report says so. + /// The stream is not parsed again, and each is reported once. /// private void CheckDeferredLengths() { @@ -3427,6 +3429,16 @@ private bool ReadSubsection(ref PdfLexer lexer, int first, int count, XRefSectio /// malformed when the rows cannot be read without it (); a /Length no section read /// so far places is read once the chain is read (). /// + /// + /// The data of such a /Length runs to its first endstream, which the file is searched for past the bytes + /// the dictionary was read with — up to the next object the sections read so far place, or the end of the file —, as + /// a stream's whose length is not confirmed, but neither stopped by an object header its rows spell nor trimmed of a + /// carriage return its last row ends in (#220). With none there, the data ends nowhere the section says, and the + /// section is malformed. Rows that spell an endstream are told from the data's end by the section's own row for + /// the /Length, when it lies before them: the data is taken again to the length the object it places gives, + /// where an endstream confirms it (). + /// bounds the dictionary alone. + /// /// private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord section, out long previous, bool candidate) { @@ -3488,13 +3500,27 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect } // A /Length no section read so far places: the data was taken up to the first endstream it holds, and the /Length is - // read, and checked, once the chain is, if the section is read. Data a guard cut is the guard's, which says so - // (ADR 34), and is not checked against a length. + // read, and checked, once the chain is, if the section is read — or before its rows are, where the section places + // it among them. Data a guard cut is the guard's, which says so (ADR 34), and is not checked against a length. var deferred = dictionary.GetRaw(PdfName.Length) is PdfReference length && ReadWithoutLoading(length.Id) is null && !section.CutByLimit && !xrefStream.Data.CutByGuard ? new DeferredLength(length.Id, xrefStream.Data.Length, number) : (DeferredLength?)null; + // Data a search found no endstream for, before the next object the sections read so far place or the end of the + // file, ends nowhere the section says: its rows are not read (#220). + if (deferred is { } unended && _endStreamSearches.TryGetValue((xrefStream.Data.Position, true), out var search) && + search.Length is null) + { + var where = search.NextObject is { } next + ? string.Create(CultureInfo.InvariantCulture, $"before the next object, at {next}") + : "before the end of the file"; + section.Fault = string.Create( + CultureInfo.InvariantCulture, + $"its /Length holds the reference {unended.Length.Number} {unended.Length.Generation} R, which no section read before it places where it can be read, and {(search.Searched ? "no endstream follows its data " + where : "its data was not searched for its endstream, the document's searches for endstream having read as much of the file as they may")}"); + return XRefSectionState.Malformed; + } + if (UnreadValue(dictionary) is { } unread) { section.Fault = unread; @@ -3554,29 +3580,6 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect return XRefSectionState.Malformed; } - var decoding = _diagnostics.GetMark(); - var decoded = Filters.PdfFilterPipeline.Decode(xrefStream, _diagnostics, out var leftEncoded); - - // Data the guard on filters left encoded holds none of the section's rows: the section is cut before its first. - var data = leftEncoded ? default : decoded.Span; - var decodedWhole = !LimitSince(_diagnostics, decoding) && !xrefStream.Data.CutByGuard; - section.CutByLimit |= !decodedWhole; - - // A row longer than the decoded data is one the data does not hold. Not a guard (ADR 34): the data's length is - // bounded by MaxDecodedStreamLength, a guard, and a valid section's rows are in its data. Each width is narrowed - // once it is known to be within the data, whose length an int holds, and their sum is then within a long. - Span fieldWidths = stackalloc int[3]; - long rowLength = 0; - - for (var i = 0; i < 3; i++) - { - fieldWidths[i] = (int)Math.Min(declaredWidths[i], data.Length + 1L); - rowLength += fieldWidths[i]; - } - - var row = rowLength <= data.Length ? (int)rowLength : 0; - var held = row == 0 ? 0 : data.Length / row; - var position = 0; long declared = size.GetValueOrDefault(); if (index is not null) @@ -3591,6 +3594,56 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect } } + // What a decoding reports of data taken again is dropped with it, and may be met again. + var decoding = deferred is null ? _diagnostics.GetMark() : _diagnostics.GetMarkWithDecodingReports(); + Span fieldWidths = stackalloc int[3]; + ReadOnlySpan data; + bool decodedWhole; + int row; + long held; + var retaken = false; + + while (true) + { + var decoded = Filters.PdfFilterPipeline.Decode(xrefStream, _diagnostics, out var leftEncoded); + + // Data the guard on filters left encoded holds none of the section's rows: the section is cut before its first. + data = leftEncoded ? default : decoded.Span; + decodedWhole = !LimitSince(_diagnostics, decoding) && !xrefStream.Data.CutByGuard; + + // A row longer than the decoded data is one the data does not hold. Not a guard (ADR 34): the data's length is + // bounded by MaxDecodedStreamLength, a guard, and a valid section's rows are in its data. Each width is narrowed + // once it is known to be within the data, whose length an int holds, and their sum is then within a long. + long rowLength = 0; + + for (var i = 0; i < 3; i++) + { + fieldWidths[i] = (int)Math.Min(declaredWidths[i], data.Length + 1L); + rowLength += fieldWidths[i]; + } + + row = rowLength <= data.Length ? (int)rowLength : 0; + held = row == 0 ? 0 : data.Length / row; + + // Fewer rows than the section declares, in data a /Length the chain could not read left to the first endstream: + // where the section's own row for that /Length, among those held, gives more data, which an endstream confirms, + // the rows spelled an endstream, and the data is taken again to that length (#220). + if (!retaken && deferred is { } unsettled && declared > held && decodedWhole && row > 0 && + RetakeDeferred(xrefStream, unsettled, data, fieldWidths, row, held, index is null ? [0, declared] : subsections) is { } whole) + { + _diagnostics.RollBack(decoding); + xrefStream = whole; + deferred = unsettled with { Taken = whole.Data.Length }; + retaken = true; + continue; + } + + break; + } + + section.CutByLimit |= !decodedWhole; + var position = 0; + // A declared count is the file's claim: the index makes room for no more rows than the data holds. ReserveIndex(Math.Min(declared, held)); @@ -3655,6 +3708,65 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect return XRefSectionState.Read; } + /// + /// Gives cross-reference stream with its data taken again to the length its /Length + /// gives, which the chain could not read and took the data up to the first endstream for, when the rows + /// holds show that endstream was in the rows (#220): the section's own row for the object + /// the /Length names, among those held, places it, and the length it gives runs past the data taken, to an + /// endstream that confirms it. Null otherwise: the length is checked once the chain is read. + /// + /// + /// The object is read where the row places it as reads one: nothing is corrected, + /// cached, recorded or reported, and one a guard cuts is not read. A row that does not stand — a newer section gave the + /// number, or refused it; the row is the second for the number, places it in an object stream, or under another + /// generation — leaves the data as it was taken. + /// + private PdfStream? RetakeDeferred( + PdfStream stream, DeferredLength deferred, ReadOnlySpan data, ReadOnlySpan widths, int row, long held, long[] subsections) + { + var number = deferred.Length.Number; + + if (_xref.TryGet(number, out _) || _refusedNumbers?.Contains(number) == true) + { + return null; + } + + // The rows before the first that numbers the object, in the order the subsections give them: their counts sum within + // a long, as the section's do. + long before = 0; + var numbered = false; + + for (var i = 0; i < subsections.Length && !numbered; i += 2) + { + numbered = number >= subsections[i] && number - subsections[i] < subsections[i + 1]; + before += numbered ? number - subsections[i] : subsections[i + 1]; + } + + if (!numbered || before >= held) + { + return null; + } + + // The row lies within the data, whose length an int holds. + var fields = data.Slice((int)before * row, row); + var cursor = 0; + var type = widths[0] == 0 ? 1 : ReadField(fields, ref cursor, widths[0]); + var offset = ReadField(fields, ref cursor, widths[1]); + var generation = ReadField(fields, ref cursor, widths[2]); + var dataStart = stream.Data.Position; + + if (type != 1 || offset > long.MaxValue || generation != (ulong)deferred.Length.Generation || + !TryReadUnrecorded(deferred.Length, PositionOf((long)offset), out var value, out _) || + value.AsInteger() is not { } length || length <= deferred.Taken || length > int.MaxValue || + length > _source.Length - dataStart || + ((IPdfStreamDataProvider)this).CheckEndStream(dataStart + length) is null) + { + return null; + } + + return new PdfStream(stream.Dictionary, ((IPdfStreamDataProvider)this).Create(dataStart, (int)length)); + } + /// /// Determines whether what an offset of the chain holds declares itself a cross-reference stream: /Type /XRef, /// or a /W. diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 5099e5e9..38e4e912 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -116,41 +116,199 @@ public static byte[] UnderAnUpdate( } /// - /// A file of one cross-reference stream, object 5, whose /Length names object 4, which only that stream - /// indexes: valid, since ISO 32000-1 lets a stream's /Length be indirect, and 7.5.8.2 does not except a - /// cross-reference stream's. Object 4 gives the data's length, plus ; with - /// , the rows of objects 6 and 7, of reserved types, spell endstream after - /// an end-of-line. With , a last row gives object 6 free with generation 13, so - /// that the data's last byte is a carriage return. The rows are compressed when , and the + /// A file of one cross-reference stream, object 5, whose /Length names an object which only that stream + /// indexes — object 4, or with the one its last row places, object 4 then free —: valid, + /// since ISO 32000-1 lets a stream's /Length be indirect, and 7.5.8.2 does not except a cross-reference stream's. + /// The length gives the data's length, plus . After the rows of objects 0 to 5 come + /// free rows; with , a row that gives the next + /// object free with generation 13, so that it ends in a carriage return; then, with , + /// two rows of reserved types that spell endstream after an end-of-line, or with + /// an object header. The rows are compressed when , and the /// data ends with before its endstream. /// public static byte[] AloneWithIndirectLength( - int lengthOff = 0, bool spellingEndStream = false, bool compressed = false, bool endingInCarriageReturn = false, string endOfLine = "\n") + int lengthOff = 0, + bool spellingEndStream = false, + bool compressed = false, + bool endingInCarriageReturn = false, + string endOfLine = "\n", + int padding = 0, + bool lengthLast = false, + bool spellingHeader = false) { using var file = new Writer("1.5"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); - var size = 6 + (spellingEndStream ? 2 : 0) + (endingInCarriageReturn ? 1 : 0); + var spelled = spellingEndStream ? "\nendstream\0\0\0\0"u8.ToArray() : spellingHeader ? "\n1 0 obj\0\0\0\0\0\0"u8.ToArray() : []; + var size = 6 + padding + (endingInCarriageReturn ? 1 : 0) + (spelled.Length / 7) + (lengthLast ? 1 : 0); + var lengthNumber = lengthLast ? size - 1 : 4; var encoding = compressed ? Rows.Flate : Rows.Raw; - // Uncompressed rows, or stored in a block of their own, take as many bytes whatever they hold. - var length = Encode(new (int, long, int)[size - (spellingEndStream ? 2 : 0)], encoding).Length + (spellingEndStream ? 14 : 0); - file.Object(4, (length + lengthOff).ToString(CultureInfo.InvariantCulture)); + // Uncompressed rows, or stored in blocks of their own, take as many bytes whatever they hold. + var length = Data(0, 0).Length + lengthOff; + file.Object(lengthNumber, length.ToString(CultureInfo.InvariantCulture)); var stream = file.Position; - (int Type, long Second, int Third)[] rows = - [(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), (1, file.OffsetOf(4), 0), (1, stream, 0)]; - var data = new List(Encode(endingInCarriageReturn ? [.. rows, (0, 0, 13)] : rows, encoding)); + var filter = compressed ? " /Filter /FlateDecode" : string.Empty; + file.Stream( + 5, + string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size {size} /W [1 4 2]{filter} /Root 1 0 R /Length {lengthNumber} 0 R"), + Data(file.OffsetOf(lengthNumber), stream), + endOfLine); + file.StartXRef(stream); + return file.ToArray(); - if (spellingEndStream) + byte[] Data(long lengthAt, long streamAt) { - // Types 10 and 101, reserved: the rows are ignored, and their bytes read "\nendstream". - data.AddRange("\nendstream\0\0\0\0"u8.ToArray()); + var rows = new List<(int Type, long Second, int Third)> + { + (0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), + lengthLast ? (0, 0, 0) : (1, lengthAt, 0), (1, streamAt, 0), + }; + + rows.AddRange(Enumerable.Repeat((0, 0L, 0), padding)); + + if (endingInCarriageReturn) + { + rows.Add((0, 0, 13)); + } + + // Types 10 and 101 or 106, reserved: the rows are ignored, and their bytes read "\nendstream", or "\n1 0 obj". The + // length's row past them is written as they are, uncompressed. + var data = new List(Encode([.. rows], encoding)); + data.AddRange(spelled); + + if (lengthLast) + { + data.AddRange(Encode([(1, lengthAt, 0)], Rows.Raw)); + } + + return [.. data]; + } + } + + /// + /// A file of two revisions. The first writes object 8 as (old) and indexes it in cross-reference stream 7. The + /// second writes it again as (new), and indexes it in cross-reference stream 10, whose /Length names + /// object 4, which only that stream indexes, and whose rows of objects 5 and 6, of reserved types, spell + /// endstream after an end-of-line when . With + /// , stream 7's /Length names object 9, which only stream 7 indexes, and a comment + /// sets the update further from it than a search for a section's misplaced offset reaches. The file is valid. + /// + public static byte[] UpdateWithIndirectLength(bool spellingEndStream, bool olderIndirect = false) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + file.Object(8, "(old)"); + var olderRows = olderIndirect ? 10 : 9; + + if (olderIndirect) + { + file.Object(9, (olderRows * 7).ToString(CultureInfo.InvariantCulture)); + } + + var first = file.Position; + (int Type, long Second, int Third)[] older = + [(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), (0, 0, 0), (0, 0, 0), (0, 0, 0), (1, first, 0), (1, file.OffsetOf(8), 0)]; + var rows = Encode(olderIndirect ? [.. older, (1, file.OffsetOf(9), 0)] : older, Rows.Raw); + var olderLength = olderIndirect ? "9 0 R" : rows.Length.ToString(CultureInfo.InvariantCulture); + file.Stream(7, string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size {olderRows} /W [1 4 2] /Root 1 0 R /Length {olderLength}"), rows); + file.StartXRef(first); + + if (olderIndirect) + { + file.Text("%" + new string('x', 2 * AdCodicem.Pdf.IO.PdfFileReader.NearbySearchRadius) + "\n"); + } + + file.Object(8, "(new)"); + var spelled = spellingEndStream ? "\nendstream\0\0\0\0"u8.ToArray() : Encode([(0, 0, 0), (0, 0, 0)], Rows.Raw); + file.Object(4, (7 + spelled.Length + 14).ToString(CultureInfo.InvariantCulture)); + var second = file.Position; + var data = new List(Encode([(1, file.OffsetOf(4), 0)], Rows.Raw)); + data.AddRange(spelled); + data.AddRange(Encode([(1, file.OffsetOf(8), 0), (1, second, 0)], Rows.Raw)); + file.Stream( + 10, + string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size 11 /W [1 4 2] /Index [4 3 8 1 10 1] /Root 1 0 R /Prev {first} /Length 4 0 R"), + [.. data]); + file.StartXRef(second); + return file.ToArray(); + } + + /// + /// A file of two revisions. The first writes object 8 as (old) and indexes it in cross-reference stream 7, whose + /// /Length names object 4, which only that stream indexes; its rows of objects 5 and 6, of reserved types, spell + /// endstream after an end-of-line when . The second, set further from it than a + /// search for a section's misplaced offset reaches, writes object 8 again as (new) and indexes it in + /// cross-reference stream 9. The file is valid. + /// + public static byte[] OlderWithIndirectLength(bool spellingEndStream) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + file.Object(8, "(old)"); + file.Object(4, "63"); + var first = file.Position; + var data = new List(Encode([(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), (1, file.OffsetOf(4), 0)], Rows.Raw)); + data.AddRange(spellingEndStream ? "\nendstream\0\0\0\0"u8.ToArray() : Encode([(0, 0, 0), (0, 0, 0)], Rows.Raw)); + data.AddRange(Encode([(1, first, 0), (1, file.OffsetOf(8), 0)], Rows.Raw)); + file.Stream(7, "/Type /XRef /Size 9 /W [1 4 2] /Root 1 0 R /Length 4 0 R", [.. data]); + file.StartXRef(first); + file.Text("%" + new string('x', 2 * AdCodicem.Pdf.IO.PdfFileReader.NearbySearchRadius) + "\n"); + + file.Object(8, "(new)"); + var second = file.Position; + var rows = Encode([(1, file.OffsetOf(8), 0), (1, second, 0)], Rows.Raw); + file.Stream( + 9, + string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size 10 /W [1 4 2] /Index [8 2] /Root 1 0 R /Prev {first} /Length {rows.Length}"), + rows); + file.StartXRef(second); + return file.ToArray(); + } + + /// + /// A file of revisions, each a cross-reference stream whose /Length names an object only + /// that stream indexes, written before it, and whose rows give that object, the stream, and + /// free objects after them; the first indexes the catalog, the page tree and the page as + /// well. The file is valid. + /// + public static byte[] UpdatesWithIndirectLengths(int count, int padding) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + var previous = -1L; + + for (var i = 0; i < count; i++) + { + var length = 4 + (i * (padding + 2)); + var rows = new List<(int Type, long Second, int Third)>(); + + if (i == 0) + { + rows.AddRange([(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0)]); + } + + file.Object(length, ((rows.Count + padding + 2) * 7).ToString(CultureInfo.InvariantCulture)); + var stream = file.Position; + rows.AddRange([(1, file.OffsetOf(length), 0), (1, stream, 0)]); + rows.AddRange(Enumerable.Repeat((0, 0L, 0), padding)); + var index = i == 0 ? $"0 {length + padding + 2}" : $"{length} {padding + 2}"; + var prev = previous < 0 ? string.Empty : string.Create(CultureInfo.InvariantCulture, $" /Prev {previous}"); + file.Stream( + length + 1, + string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size {length + padding + 2} /W [1 4 2] /Index [{index}] /Root 1 0 R{prev} /Length {length} 0 R"), + Encode([.. rows], Rows.Raw)); + file.StartXRef(stream); + previous = stream; } - var filter = compressed ? " /Filter /FlateDecode" : string.Empty; - file.Stream(5, string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size {size} /W [1 4 2]{filter} /Root 1 0 R /Length 4 0 R"), [.. data], endOfLine); - file.StartXRef(stream); return file.ToArray(); } diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index db331c6d..4dd82879 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -5,7 +5,9 @@ using AdCodicem.Pdf.Diagnostics; using AdCodicem.Pdf.Documents; using AdCodicem.Pdf.IO; +using AdCodicem.Pdf.IO.XRef; using AdCodicem.Pdf.Objects; +using AdCodicem.Pdf.Validation; namespace AdCodicem.Pdf.Tests; @@ -1818,23 +1820,235 @@ public void Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_le document.GetObject(new PdfObjectId(3)).AsDictionary().IsOfType(PdfName.Page).Should().BeTrue(); } + [Theory] + [InlineData(0)] + [InlineData(9_500)] + public void Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives(int padding) + { + // Two rows of reserved types spell an endstream after an end-of-line, a few bytes in or past the first 64 KB: the + // chain, which could not read the /Length, took the data up to it, and read the rows before it alone — once with a + // report, the section said to lack rows. The stream's own row for object 4 lies among them: the object it places + // gives the length, which the endstream after it confirms, and the data is taken again to it (#220). + var file = ChainFiles.AloneWithIndirectLength(spellingEndStream: true, padding: padding); + using var document = PdfDocument.Open(file); + + document.Diagnostics.Should().BeEmpty(); + document.WasRepaired.Should().BeFalse(); + document.Reader.Index.Count.Should().Be(6 + padding); + document.Reader.Structure.Sections.Should().ContainSingle().Which.Incomplete.Should().BeFalse(); + new PdfValidator().Validate(document).Findings.Should().BeEmpty(); + } + + [Fact] + public void Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes() + { + // The update's stream indexes its /Length, object 4, and object 8 after rows that spell an endstream: read up to it, + // the stream held object 4's row alone, and object 8 read as the first revision wrote it, a valid file said to have + // a malformed section. The update's rows are read whole, and object 8 reads as it does in its direct twin (#220). + using var document = PdfDocument.Open(ChainFiles.UpdateWithIndirectLength(spellingEndStream: true)); + using var twin = PdfDocument.Open(ChainFiles.UpdateWithIndirectLength(spellingEndStream: false)); + + document.GetObject(new PdfObjectId(8)).Should().BeOfType().Which.ToText().Should().Be("new"); + twin.GetObject(new PdfObjectId(8)).Should().BeOfType().Which.ToText().Should().Be("new"); + document.Diagnostics.Should().BeEmpty(); + document.Reader.Index.Count.Should().Be(twin.Reader.Index.Count); + new PdfValidator().Validate(document).Findings.Should().BeEmpty(); + } + [Fact] public void Reports_the_rows_a_deferred_length_shows_the_chain_did_not_read() { - // The rows of objects 6 and 7, of reserved types, spell an endstream after an end-of-line: the chain, which could - // not read the /Length, took the data up to it. The length, read once the chain is, gives 14 bytes more, which the - // endstream after them confirms: what the chain left unread is reported. The rows are not read again. - var file = ChainFiles.AloneWithIndirectLength(spellingEndStream: true); + // The rows spell an endstream before the stream's own row for its /Length, the last: what the chain read holds no + // row to place it by. The length, read once the chain is, gives 21 bytes more, which the endstream after them + // confirms: what the chain left unread is reported, and the section's missing rows set off a rebuild. + var file = ChainFiles.AloneWithIndirectLength(spellingEndStream: true, lengthLast: true); using var document = PdfDocument.Open(file); - var report = document.Diagnostics.Should().ContainSingle().Which; - report.Code.Should().Be(PdfDiagnosticCodes.StreamLengthInvalid); + var report = document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.StreamLengthInvalid).Which; report.Position.Should().Be(PdfTemplate.OffsetOf(file, "stream\n", 1) + "stream\n".Length); report.Message.Should().Be( - "The /Length of cross-reference stream 5 gives 56 bytes, which an endstream confirms; the chain, which could not read it, took the data up to an endstream 42 bytes in, and read no row past it."); + "The /Length of cross-reference stream 5 gives 63 bytes, which an endstream confirms; the chain, which could not read it, took the data up to an endstream 42 bytes in, and read no row past it."); + document.Reader.Structure.Sections.Should().ContainSingle().Which.Incomplete.Should().BeTrue(); document.GetObject(new PdfObjectId(3)).AsDictionary().IsOfType(PdfName.Page).Should().BeTrue(); } + [Theory] + [InlineData("generation")] + [InlineData("object stream")] + [InlineData("name")] + [InlineData("short")] + [InlineData("unconfirmed")] + [InlineData("unreadable")] + [InlineData("superseded")] + public void A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell(string fault) + { + // The stream's row for object 4 places it under another generation, or in an object stream; object 4 holds a name; + // it gives no more than the data taken, or more that no endstream confirms; its header is misspelled; an update + // gives object 4 a place that holds nothing. None says where the data ends: the rows before the endstream they + // spell are all the section holds, as the chain took them (#220). + var file = ChainFiles.AloneWithIndirectLength( + spellingEndStream: true, lengthOff: fault switch { "short" => -14, "unconfirmed" => 3, _ => 0 }); + var row = (int)PdfTemplate.OffsetOf(file, "stream\n", 1) + "stream\n".Length + (4 * 7); + + switch (fault) + { + case "generation": + file[row + 6] = 1; + break; + case "object stream": + file[row] = 2; + break; + case "name": + case "unreadable": + var replaced = Encoding.Latin1.GetString(file).Replace( + "4 0 obj\n56\n", fault == "name" ? "4 0 obj\n/F\n" : "4 0 xbj\n56\n", StringComparison.Ordinal); + file = Encoding.Latin1.GetBytes(replaced); + break; + case "superseded": + file = WithUpdatePlacingFourOutside(file); + break; + } + + using var document = PdfDocument.Open(file); + + document.Reader.Structure.Sections[^1].Incomplete.Should().BeTrue(); + document.Reader.Structure.Sections[^1].Fault.Should().Be("it holds 6 rows where its /Index and /Size declare 8"); + + static byte[] WithUpdatePlacingFourOutside(byte[] file) + { + var stream = PdfTemplate.OffsetOf(file, "5 0 obj"); + var update = new StringBuilder("%").Append('x', 2 * PdfFileReader.NearbySearchRadius).Append('\n'); + var xref = file.Length + update.Length; + update.Append(CultureInfo.InvariantCulture, $"xref\n4 1\n0000099999 00000 n\r\ntrailer\n<< /Size 8 /Root 1 0 R /Prev {stream} >>\nstartxref\n{xref}\n%%EOF\n"); + return [.. file, .. Encoding.Latin1.GetBytes(update.ToString())]; + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives(bool twin) + { + // The older section's /Length names object 4, which only it indexes, and two of its rows spell an endstream: the + // search for it stops at the update's first object, past them. Its data is taken again to the length object 4 + // gives, as the newest section's is (#220). + var file = ChainFiles.OlderWithIndirectLength(spellingEndStream: !twin); + using var document = PdfDocument.Open(file); + + document.Diagnostics.Should().BeEmpty(); + document.WasRepaired.Should().BeFalse(); + document.Reader.Structure.Sections.Should().HaveCount(2).And.OnlyContain(section => !section.Incomplete); + document.GetObject(new PdfObjectId(8)).Should().BeOfType().Which.ToText().Should().Be("new"); + new PdfValidator().Validate(document).Findings.Should().BeEmpty(); + } + + [Fact] + public void Updates_each_with_a_long_deferred_length_are_all_searched() + { + // Five revisions, each a cross-reference stream of 70,000 bytes of rows whose /Length only it indexes: each is + // searched for its endstream past the first window, up to the next object the newer sections place or the end of + // the file. Together they read about the file once, and one read past each: no search is left undone for want of + // what the document's searches may read (EndStreamSearchPasses). + var file = ChainFiles.UpdatesWithIndirectLengths(count: 5, padding: 10_000); + using var source = new StrictCountingSource(file); + using var document = PdfDocument.Open(source, options: null, ownsSource: false); + + document.Diagnostics.Should().BeEmpty(); + document.Reader.Structure.Sections.Should().HaveCount(5).And.OnlyContain(section => section.State == XRefSectionState.Read && !section.Incomplete); + document.Reader.Index.Count.Should().Be(4 + (5 * 10_002)); + source.BytesRead.Should().BeLessThan(3L * file.Length); + } + + [Theory] + [InlineData(false, false, "\n")] + [InlineData(true, false, "\n")] + [InlineData(false, true, "\r\n")] + public void A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin( + bool compressed, bool endingInCarriageReturn, string endOfLine) + { + // 70,000 bytes of rows, past the 64 KB the chain's window first holds: the data, whose /Length object 4 gives and + // only the stream indexes, was searched for its endstream within the window alone, which grew to MaxTrailerLength, + // reported reached, and 9,350 of 10,000 rows were read. It is searched in the file, as any stream's whose length is + // not confirmed, and read whole — its last byte a carriage return too (#220). + var file = ChainFiles.AloneWithIndirectLength(padding: 10_000, compressed: compressed, endingInCarriageReturn: endingInCarriageReturn, endOfLine: endOfLine); + var options = new PdfReaderOptions { ThrowOnLimit = true }; + using var document = PdfDocument.Open(file, options); + + document.Diagnostics.Should().BeEmpty(); + document.WasRepaired.Should().BeFalse(); + document.Reader.Index.Count.Should().Be(10_006 + (endingInCarriageReturn ? 1 : 0)); + document.Reader.Structure.Sections.Should().ContainSingle().Which.CutByLimit.Should().BeFalse(); + } + + [Fact] + public void A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s() + { + // The search reads the data to its endstream, and the decoding reads it again: the file is read once more than its + // direct twin, whose /Length, written in place of "4 0 R", takes as many bytes, and not a byte past the next read. + var file = ChainFiles.AloneWithIndirectLength(padding: 10_000); + var text = Encoding.Latin1.GetString(file); + var twin = Encoding.Latin1.GetBytes(text.Replace("/Length 4 0 R", "/Length 70042", StringComparison.Ordinal)); + twin.Length.Should().Be(file.Length); + + var read = BytesReadOpening(file); + var twinRead = BytesReadOpening(twin); + + read.Should().BeInRange(twinRead, twinRead + 70_042 + PdfFileReader.EndStreamSearchChunk); + + static long BytesReadOpening(byte[] file) + { + using var source = new StrictCountingSource(file); + using var document = PdfDocument.Open(source, options: null, ownsSource: false); + document.Diagnostics.Should().BeEmpty(); + return source.BytesRead; + } + } + + [Fact] + public void Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data() + { + // Rows of reserved types spell "1 0 obj" past the first 64 KB: a header the bytes hold ends a search for a stream's + // endstream, but not one whose /Length the chain could not read, whose rows may spell anything (#220). + using var document = PdfDocument.Open(ChainFiles.AloneWithIndirectLength(padding: 10_000, spellingHeader: true)); + + document.Diagnostics.Should().BeEmpty(); + document.Reader.Index.Count.Should().Be(10_006); + } + + [Fact] + public void A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed() + { + // The stream's endstream is misspelled: its data, which no length the chain could read ends, ends nowhere, and the + // section is not read — once, the rest of the file was taken for its data (#220). The index is rebuilt. + var file = ChainFiles.AloneWithIndirectLength(padding: 10_000); + var at = (int)PdfTemplate.OffsetOf(file, "endstream"); + file[at] = (byte)'x'; + using var document = PdfDocument.Open(file); + + document.WasRepaired.Should().BeTrue(); + var section = document.Reader.Structure.Sections.Should().ContainSingle().Which; + section.State.Should().Be(XRefSectionState.Malformed); + section.Fault.Should().Be( + "its /Length holds the reference 4 0 R, which no section read before it places where it can be read, and no endstream follows its data before the end of the file"); + document.Diagnostics.Select(entry => entry.Code).Should().NotContain(code => code.StartsWith("limit.", StringComparison.Ordinal)); + } + + [Fact] + public void A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed() + { + // The older stream's /Length names object 9, which only it indexes, and its endstream is misspelled: the search for + // it stops at the update's first object the newer section places (#220). + var file = ChainFiles.UpdateWithIndirectLength(spellingEndStream: false, olderIndirect: true); + var at = (int)PdfTemplate.OffsetOf(file, "endstream"); + file[at] = (byte)'x'; + var next = PdfTemplate.OffsetOf(file, "8 0 obj\n(new)"); + using var document = PdfDocument.Open(file); + + document.Reader.Structure.Sections[1].Fault.Should().Be(string.Create( + CultureInfo.InvariantCulture, + $"its /Length holds the reference 9 0 R, which no section read before it places where it can be read, and no endstream follows its data before the next object, at {next}")); + } + [Theory] [MemberData(nameof(ChainFiles.Keys), MemberType = typeof(ChainFiles))] public void Changes_nothing_of_the_index_while_the_chain_is_read(string key) diff --git a/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs b/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs index 7f07b965..84116851 100644 --- a/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs +++ b/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs @@ -19,6 +19,7 @@ public class HostileInputTests { private static readonly TimeSpan Budget = TimeSpan.FromSeconds(10); + [Theory] [InlineData("/A 1 ")] [InlineData("/A null ")] @@ -276,6 +277,24 @@ public void A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_th document.Reader.IsEndObjMissing(5, out _).Should().BeTrue(); } + [Fact] + public void A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times() + { + // A cross-reference stream whose /Length only it indexes, 3 MB of rows, and its endstream misspelled: its search + // reads the data to the end of the file, and its section is malformed — the file's fault, no guard's (#220). The + // rebuild scans the file, and its stream, its length read then, is searched again: the file is read about four + // times, as its sound twin is twice — the search and the decoding. + var file = ChainFiles.AloneWithIndirectLength(padding: 450_000); + file[(int)PdfTemplate.OffsetOf(file, "endstream")] = (byte)'x'; + using var source = new StrictCountingSource(file); + + using var document = PdfDocument.Open(source, options: null, ownsSource: false); + + document.Reader.Structure.Sections.Should().ContainSingle().Which.State.Should().Be(XRefSectionState.Malformed); + document.Diagnostics.Should().NotContain(d => d.Code.StartsWith("limit.", StringComparison.Ordinal)); + source.BytesRead.Should().BeLessThan(5L * file.Length); + } + [Fact] public void Reading_streams_whose_data_share_a_stretch_of_a_file_held_in_memory_allocates_none_of_their_data() { diff --git a/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs b/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs index 0a87176c..ebfcbdbb 100644 --- a/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs @@ -56,6 +56,21 @@ public void Unbounded_takes_every_guard_to_the_most_the_implementation_holds() limits.MaxFilterChainLength.Should().Be(int.MaxValue); } + [Fact] + public void MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data() + { + // The data of a cross-reference stream whose /Length only it indexes was searched for its endstream within the + // window that read the dictionary, which MaxTrailerLength bounds: a kilobyte of rows past 256 bytes reached the guard. + // The data is searched in the file, as any stream's whose length is not confirmed (#220). + var file = ChainFiles.AloneWithIndirectLength(padding: 150); + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxTrailerLength = 256 }, ThrowOnLimit = true }; + + using var document = PdfDocument.Open(file, options); + + document.Diagnostics.Should().BeEmpty(); + document.Reader.Index.Count.Should().Be(156); + } + [Theory] [InlineData(nameof(PdfReaderLimits.MaxDecodedStreamLength), 0)] [InlineData(nameof(PdfReaderLimits.MaxStreamLength), 0)] From 79bc02dc1b188e4c0fa19ad12166e6bf55333ee5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 06:54:26 +0000 Subject: [PATCH 006/146] fix: take a stream's /Length that a guard cut for the guard's, not the file's A /Length naming an object a guard cut was judged by what the cut left of it: null, reported as a /Length that holds no length, or an integer the cut parted, 1 of 15, reported as a declared length the data contradicts. Neither is a value the file gave. The answer is keyed on the object the reference leads to, through a chain of references too, and never on the value read: one a guard cut takes a form of its own, and one a guard left out of the index the existing one. An integer read before the cut is only a place to look for endstream at, never kept as a length; the data is otherwise found by its endstream, and nothing is said of the length. A missing endstream is still the file's, and reported. A cross-reference stream's /Length read once the chain is read is left alike. No message is made for a length not judged. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../IO/IPdfStreamDataProvider.cs | 5 +- src/AdCodicem.Pdf/IO/ObjectPresence.cs | 8 + src/AdCodicem.Pdf/IO/PdfFileReader.cs | 43 ++++- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 20 +- src/AdCodicem.Pdf/IO/StreamLengthForm.cs | 8 + .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 181 ++++++++++++++++++ 6 files changed, 251 insertions(+), 14 deletions(-) create mode 100644 tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs diff --git a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs index d3c200dd..0091658c 100644 --- a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs +++ b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs @@ -59,9 +59,10 @@ internal interface IPdfStreamDataProvider /// /// Resolves the object a stream's /Length names, and says whether the file holds it and whether it could be - /// read — a reference to the stream itself, or to an object inside an object stream being decoded, cannot be. + /// read — a reference to the stream itself, or to an object inside an object stream being decoded, cannot be —, or + /// whether a guard cut it, or the object a chain of references from it leads to (#224). /// - PdfObject ResolveLength(PdfObjectId id, out ObjectPresence presence); + PdfObject ResolveLength(PdfReference reference, out ObjectPresence presence); /// /// Determines whether what is wrong with the length of the stream whose data starts at diff --git a/src/AdCodicem.Pdf/IO/ObjectPresence.cs b/src/AdCodicem.Pdf/IO/ObjectPresence.cs index 7f89e7ca..026f16ac 100644 --- a/src/AdCodicem.Pdf/IO/ObjectPresence.cs +++ b/src/AdCodicem.Pdf/IO/ObjectPresence.cs @@ -31,4 +31,12 @@ internal enum ObjectPresence : byte /// Whether the file holds the object is unknown, and a reference to it reads as null until the guard is lifted. /// Unindexed, + + /// + /// The object ran past one of the reader's guards, which cut it: what it holds may lie past the cut, and what was read + /// of it — null, or a token the cut parted — is the guard's until the guard is lifted (#224). Only a stream's + /// /Length asks: says whether the file holds the object, and + /// whether a guard cut it. + /// + Cut, } diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index d140394b..261c7915 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -1380,6 +1380,18 @@ private static bool Consume(HeaderReading reading, ref long consumed, long end) /// internal bool IsCutAtLimit(int number) => _cutAtLimit.Contains(number); + /// + /// Determines whether the object leads to — through a chain of references too, as + /// follows it — is one a guard cut, or left out of the index: what it holds may lie + /// past the cut, or is unknown, and is the guard's until the guard is lifted, rather than a value the file gave + /// (#224, #263). Call it once the reference is resolved. + /// + internal bool LeftByGuard(PdfReference reference) + { + var target = reference.Target().Number; + return IsCutAtLimit(target) || GetPresence(target) == ObjectPresence.Unindexed; + } + /// /// Determines whether object , as it was read, is a stream whose length the file did not /// confirm, and gives what the reader found of it. @@ -1810,8 +1822,10 @@ StreamEndSearch IPdfStreamDataProvider.FindEndStream( } /// - PdfObject IPdfStreamDataProvider.ResolveLength(PdfObjectId id, out ObjectPresence presence) + PdfObject IPdfStreamDataProvider.ResolveLength(PdfReference reference, out ObjectPresence presence) { + var id = reference.Id; + if (_readingChain) { // A /Length no section read so far places is read once the chain is read: the data is found by its endstream @@ -1826,6 +1840,18 @@ PdfObject IPdfStreamDataProvider.ResolveLength(PdfObjectId id, out ObjectPresenc // A null a guard against re-entry gave says nothing of the object: it could not be read from here. presence = value is PdfNull && reentries != _reentries ? ObjectPresence.Unproduced : GetPresence(id.Number); + + // What the object holds, or the one a chain of references from it leads to, is the guard's when a guard cut it or + // left it out of the index: a value the cut left, null or an integer it parted, is none the file gave (#224). + if (presence != ObjectPresence.Unindexed) + { + // The object is not loaded again unless its value leads further. + var target = value is PdfReference ? reference.Target().Number : id.Number; + presence = IsCutAtLimit(target) ? ObjectPresence.Cut + : target != id.Number && GetPresence(target) == ObjectPresence.Unindexed ? ObjectPresence.Unindexed + : presence; + } + return value; } @@ -2118,16 +2144,17 @@ private void CheckDeferredLengths() var (number, generation) = (deferred.Length.Number, deferred.Length.Generation); string? message = null; + // A /Length a guard cut or left out of the index — or the object a chain of references from it leads to — is + // unknown, not wrong: the guard's report says why, and what the cut left of it, null or a parted integer, is + // none the file gave (#224). + if (LeftByGuard(new PdfReference(deferred.Length, this))) + { + continue; + } + if (value.AsInteger() is not { } length || length is < 0 or > int.MaxValue) { var presence = value is PdfNull && reentries != _reentries ? ObjectPresence.Unproduced : GetPresence(number); - - // A /Length a guard left out of the index is unknown, not wrong: the guard's report says why. - if (presence == ObjectPresence.Unindexed) - { - continue; - } - var held = presence switch { ObjectPresence.Missing => "which the file lacks", diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index e9f80bb3..52de4911 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -713,7 +713,9 @@ private PdfStream ReadStream(PdfDictionary dictionary) return Finish(dictionary, dataStart, Math.Max(length, 0), span.Length); } - var confirmed = ConfirmsLength(span, dataStart, length); + // An integer read from a /Length a guard cut is a place to check, never a length kept when nothing confirms it. + var checkedAt = declared.Form == StreamLengthForm.ObjectCut ? (int)declared.Value.GetValueOrDefault(-1) : length; + var confirmed = ConfirmsLength(span, dataStart, checkedAt); if (confirmed is null) { @@ -724,7 +726,7 @@ private PdfStream ReadStream(PdfDictionary dictionary) if (confirmed == true) { - return Finish(dictionary, dataStart, length, span.Length); + return Finish(dictionary, dataStart, checkedAt, span.Length); } // A cross-reference stream whose /Length the chain could not read keeps a carriage return before the line feed as @@ -744,7 +746,9 @@ private PdfStream ReadStream(PdfDictionary dictionary) if (settled.Length is { } found) { - if (length != found) + // A length read once the chain is, or one a guard left out of the index or cut, is not judged: nothing is said + // of it, nor a message made for it. + if (length != found && declared.Form is not (StreamLengthForm.Deferred or StreamLengthForm.ObjectUnindexed or StreamLengthForm.ObjectCut)) { ReportLengthFault( PdfDiagnosticCodes.StreamLengthInvalid, @@ -954,7 +958,7 @@ private readonly DeclaredLength ReadLength(PdfDictionary dictionary) if (_streamData is not null) { - value = _streamData.ResolveLength(named.Id, out presence).Resolve(); + value = _streamData.ResolveLength(named, out presence).Resolve(); } else { @@ -964,6 +968,13 @@ private readonly DeclaredLength ReadLength(PdfDictionary dictionary) presence = value is PdfNull ? ObjectPresence.Unproduced : ObjectPresence.Defined; } + // An integer the cut left, whole or parted, is only a place to look for endstream at (#224). + if (presence == ObjectPresence.Cut) + { + return new DeclaredLength( + StreamLengthForm.ObjectCut, value.AsInteger() is { } hint and >= 0 and <= int.MaxValue ? hint : null, reference); + } + if (presence != ObjectPresence.Defined) { return new DeclaredLength( @@ -1034,6 +1045,7 @@ private static string Describe(DeclaredLength declared) { StreamLengthForm.ObjectMissing => Names(reference, "which the file lacks"), StreamLengthForm.ObjectUnreadable => Names(reference, "which could not be read"), + StreamLengthForm.ObjectCut => Names(reference, "which one of the reader's limits cut"), _ => Names(reference, "which holds " + Value(declared)), }; diff --git a/src/AdCodicem.Pdf/IO/StreamLengthForm.cs b/src/AdCodicem.Pdf/IO/StreamLengthForm.cs index 78cadac7..4934ece9 100644 --- a/src/AdCodicem.Pdf/IO/StreamLengthForm.cs +++ b/src/AdCodicem.Pdf/IO/StreamLengthForm.cs @@ -36,4 +36,12 @@ internal enum StreamLengthForm : byte /// data is taken up to its endstream, and nothing is reported: the guard's report says why. /// ObjectUnindexed, + + /// + /// A reference to an object one of the reader's guards cut, or that a chain of references from it leads to: what it + /// holds may lie past the cut. An integer read before the cut is only a place to look for endstream at, and + /// never kept as a length; the data is otherwise taken up to its endstream. Nothing is reported of the length, + /// whose fault is the guard's, which reported it; a missing endstream is still the file's (#224). + /// + ObjectCut, } diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs new file mode 100644 index 00000000..6e07e149 --- /dev/null +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -0,0 +1,181 @@ +using System.Globalization; +using System.Text; +using AdCodicem.Pdf.Diagnostics; +using AdCodicem.Pdf.Documents; +using AdCodicem.Pdf.IO; +using AdCodicem.Pdf.Objects; +using AdCodicem.Pdf.Validation; + +namespace AdCodicem.Pdf.Tests; + +/// +/// A stream's /Length, /Filter or /DecodeParms that leads to an object one of the reader's guards cut, +/// or left out of the index: what the cut left of it — null, a token it parted, a whole value it may yet go on past — is +/// the guard's, which reported it, and not a value the file gave (#224, #263). +/// +public class GuardCutValueTests +{ + private const string Page = "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Contents 4 0 R /Resources << >> >>"; + + private static readonly PdfReaderOptions ObjectGuard = new() { Limits = PdfReaderLimits.Default with { MaxObjectLength = 1024 } }; + + [Theory] + [InlineData("before the value", "0 0 m")] + [InlineData("after the value", "0 0 m")] + [InlineData("after a wrong value", "0 0 m")] + [InlineData("inside the value", "0 0 m 10 10 l S")] + public void A_length_a_guard_cut_is_not_judged_and_the_data_is_found_by_its_endstream(string cut, string data) + { + // Object 20, the /Length, runs past MaxObjectLength: before its value, which read as null; after it, or after a + // wrong one; or inside "15", which read as 1. Each was reported as the file's fault but the one after the right + // value — "holds null", "declared 4 bytes", "declared 1 bytes" (#224). The data is found by its endstream, and only + // the guard is reported. + var twenty = cut switch + { + "before the value" => new string(' ', 3000) + "5", + "after the value" => "5" + new string(' ', 3000), + "after a wrong value" => "4" + new string(' ', 3000), + _ => new string(' ', 1024 - "20 0 obj\n".Length - 1) + "15" + new string(' ', 100), + }; + var file = ContentLengthIn(twenty, data); + using var document = PdfDocument.Open(file, ObjectGuard); + + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + stream.GetRawBytes().ToArray().Should().Equal(Encoding.ASCII.GetBytes(data)); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.Reader.TryGetStreamLengthFault(4, out _).Should().BeFalse(); + document.Reader.IsCutAtLimit(20).Should().BeTrue(); + + // Read whole, a wrong value is the file's. + using var control = PdfDocument.Open(file); + control.GetObject(new PdfObjectId(4)).AsStream().Required().GetRawBytes().ToArray().Should().Equal(Encoding.ASCII.GetBytes(data)); + control.Diagnostics.Select(entry => entry.Message).Should().Equal( + cut == "after a wrong value" ? ["The stream declared 4 bytes but ended after 5."] : []); + } + + [Fact] + public void A_length_a_guard_cut_leaves_a_missing_endstream_the_file_s_fault() + { + // The stream lost its endstream: its own endobj ends its data, which is the file's to answer for, whatever its + // length — the guard cut that, and is not judged. + var file = ContentLengthIn(new string(' ', 3000) + "5", "0 0 m", endStream: false); + using var document = PdfDocument.Open(file, ObjectGuard); + + document.GetObject(new PdfObjectId(4)).AsStream().Required().GetRawBytes().ToArray().Should().Equal("0 0 m"u8.ToArray()); + var report = document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.StreamTruncated).Which; + report.Message.Should().StartWith( + "The stream's /Length names object 20 0, which one of the reader's limits cut, and no endstream follows its data"); + report.Message.Should().EndWith("its object's endobj ends it, after 5 bytes."); + } + + [Fact] + public void A_length_an_object_stream_s_guard_left_unread_is_not_judged() + { + // Object 20, the /Length, is a member of object stream 11 past what MaxDecodedStreamLength lets it decode to: it + // read as null, and was reported as one that could not be read (#263). It is the guard's. + var file = new HandWrittenFile(version: "1.5"); + var rows = new Dictionary + { + [1] = (1, file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), 0), + [2] = (1, file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), 0), + [3] = (1, file.Object(3, Page), 0), + [4] = (1, file.Object(4, "<< /Length 20 0 R >>\nstream\n0 0 m\nendstream"), 0), + [11] = (1, FlateObjectStream(file, 11, [(21, "(" + new string('x', 300) + ")"), (20, "5")]), 0), + [21] = (2, 11, 0), + [20] = (2, 11, 1), + }; + file.CrossReferenceStream(12, 22, rows, "/Root 1 0 R"); + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxDecodedStreamLength = 100 } }; + using var document = PdfDocument.Open(file.Bytes, options); + + document.GetObject(new PdfObjectId(4)).AsStream().Required().GetRawBytes().ToArray().Should().Equal("0 0 m"u8.ToArray()); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitDecodedStream); + document.Reader.TryGetStreamLengthFault(4, out _).Should().BeFalse(); + } + + [Fact] + public void A_length_whose_reference_leads_to_an_object_a_guard_left_out_of_the_index_is_not_judged() + { + // Object 20, the /Length, holds a reference to object 21, which only an older section places, which + // MaxXRefSectionCount leaves unread — the newer table numbers no row 21: 21 is unknown, and 20 led to null, + // reported as no length. + var file = new HandWrittenFile(); + var older = new Dictionary { [21] = file.Object(21, "5") }; + var first = file.Table(22, older, "/Size 22"); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, Page), + [4] = file.Object(4, "<< /Length 20 0 R >>\nstream\n0 0 m\nendstream"), + [20] = file.Object(20, "21 0 R"), + }; + file.Table(21, rows, string.Create(CultureInfo.InvariantCulture, $"/Size 22 /Root 1 0 R /Prev {first}")); + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxXRefSectionCount = 1 } }; + using var document = PdfDocument.Open(file.Bytes, options); + + document.GetObject(new PdfObjectId(4)).AsStream().Required().GetRawBytes().ToArray().Should().Equal("0 0 m"u8.ToArray()); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitXRefSectionCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read(bool inside) + { + // The cross-reference stream's /Length, object 5, which the newer table places, runs past MaxObjectLength before its + // value, or inside "56": read once the chain is, it was said to hold null, or to declare 5 bytes (#224). + var five = inside ? new string(' ', 1024 - "5 0 obj\n".Length - 1) + "56" + new string(' ', 100) : new string(' ', 3000) + "56"; + var file = ChainFiles.UnderAnUpdate("/Type /XRef /Size 8 /W [1 4 2] /Length 5 0 R", five, compressed: false, predicted: false, placed: true); + using var document = PdfDocument.Open(file, ObjectGuard); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.WasRepaired.Should().BeFalse(); + document.Reader.Structure.Sections.Should().OnlyContain(section => section.State == IO.XRef.XRefSectionState.Read); + } + + /// + /// A page whose content stream, object 4, holds , its /Length object 20, which holds + /// ; with no endstream unless . + /// + private static byte[] ContentLengthIn(string twenty, string data, bool endStream = true) + { + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, Page), + [4] = file.Object(4, "<< /Length 20 0 R >>\nstream\n" + data + (endStream ? "\nendstream" : string.Empty)), + [20] = file.Object(20, twenty), + }; + file.Table(21, rows, "/Size 21 /Root 1 0 R"); + return file.Bytes; + } + + /// + /// Writes object stream , holding compressed with Flate, or with the + /// filter names, its dictionary holding before its own. + /// + private static long FlateObjectStream( + HandWrittenFile file, int number, (int Number, string Body)[] members, string entries = "", string filter = "/FlateDecode") + { + var header = new StringBuilder(); + var contents = new StringBuilder(); + + foreach (var (member, body) in members) + { + header.Append(CultureInfo.InvariantCulture, $"{member} {contents.Length} "); + contents.Append(body).Append(' '); + } + + var data = HostileInputTests.Compress(Encoding.ASCII.GetBytes(header.ToString() + contents)); + var first = entries.Contains("/First", StringComparison.Ordinal) ? string.Empty : string.Create(CultureInfo.InvariantCulture, $" /First {header.Length}"); + return file.Object( + number, + string.Create( + CultureInfo.InvariantCulture, + $"<< /Type /ObjStm /N {members.Length}{first} {entries}/Filter {filter} /Length {data.Length} >>\nstream\n{Encoding.Latin1.GetString(data)}\nendstream")); + } +} From cd219bd78f74805838e11f446073a8ad4a035a95 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 06:54:36 +0000 Subject: [PATCH 007/146] fix: end a stream's filters at a value a guard cut or left out of the index A /Filter, a /DecodeParms, or a value of a step's parameters that led to an object a guard cut, or left out of the index, read as no filter, or as "/Flate" of "/FlateDecode" parted: an object stream decoded so read its header as stray bytes, the catalog it held was looked for elsewhere, and a sound file under a lowered limit was judged broken. The chain of filters now ends at such a value, as at the element past MaxFilterChainLength: the steps before it run, the data is left encoded, and an object stream so left is hidden and its members taken as the guard's. A guard that loading a filter's value reached was taken for a cut of the stream's own data, which hid the stream's own faults, and made the reader and the validator disagree as read order went. Decoding now says whether the stream's own data is cut, which is what the readers of rows and members ask. While the chain is read, a cross-reference stream's /W, /Index, /Size, /Filter or /DecodeParms naming an object MaxObjectLength cuts made the section malformed, and the index was rebuilt. The section is now the guard's: the chain ends there, nothing is rebuilt, and the guard is reached once the chain is read, thrown at opening when the caller asks. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/Filters/FilterChain.cs | 53 ++++- .../IO/Filters/FilterChainEnd.cs | 7 + .../IO/Filters/PdfFilterPipeline.cs | 45 ++-- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 99 ++++++--- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 197 ++++++++++++++++++ 5 files changed, 352 insertions(+), 49 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/Filters/FilterChain.cs b/src/AdCodicem.Pdf/IO/Filters/FilterChain.cs index a571e5a4..df6b6078 100644 --- a/src/AdCodicem.Pdf/IO/Filters/FilterChain.cs +++ b/src/AdCodicem.Pdf/IO/Filters/FilterChain.cs @@ -32,6 +32,13 @@ namespace AdCodicem.Pdf.IO.Filters; /// towards it, though decoding runs no step for it. /// /// +/// It ends, as well, at a value that leads to an object one of the reader's guards cut, or left out of the index: the +/// /Filter entry or an element of it, /DecodeParms or a value of a step's parameters. What the cut left of +/// it — null, a name it parted, a value it may yet go on past — is the guard's, and no filter the file gave: the step it +/// is read for, and those after it, do not run, and the data is left as the steps before them decode it, the guard +/// reported where it was reached (#263). +/// +/// /// What was resolved is kept only where the file did not write it plainly. The /Filter array is read again as the /// steps run, so a name it holds costs nothing more; the name an element written as a reference led to is kept, a slot /// each. A dictionary that serves every step is read once for Flate and once for LZW, and of an array of parameters, only @@ -107,9 +114,8 @@ private FilterChain(in Reading reading) /// The most filters the stream's data may pass through: the chain ends at the element past them. public static FilterChain Resolve(PdfDictionary dictionary, int maxFilters) { - var reader = default(Resolving); - TryRead(dictionary, maxFilters, ref reader, readParameters: true, keep: true, out var reading); - return reading.Filters is null ? NoFilter : new FilterChain(reading); + var reading = Read(dictionary, maxFilters, readParameters: true, keep: true); + return reading.Filters is null && reading.End == FilterChainEnd.None ? NoFilter : new FilterChain(reading); } /// @@ -118,12 +124,8 @@ public static FilterChain Resolve(PdfDictionary dictionary, int maxFilters) /// /// The stream's dictionary. /// The most filters the stream's data may pass through: the chain ends at the element past them. - public static FilterChainEnd EndOf(PdfDictionary dictionary, int maxFilters) - { - var reader = default(Resolving); - TryRead(dictionary, maxFilters, ref reader, readParameters: false, keep: false, out var reading); - return reading.End; - } + public static FilterChainEnd EndOf(PdfDictionary dictionary, int maxFilters) => + Read(dictionary, maxFilters, readParameters: false, keep: false).End; /// /// Follows, through , every value decoding the stream describes @@ -327,6 +329,27 @@ PdfName name when PdfFilterPipeline.IsImageFilter(name) => FilterChainEnd.Image, return true; } + /// + /// Resolves the chain of filters the stream describes, as decoding reads it: a value that + /// leads to an object one of the reader's guards cut, or left out of the index, ends it where it is read, as the + /// element past the guard on filters does — the steps before it run, and what they decode is left encoded by the rest + /// (#263). + /// + private static Reading Read(PdfDictionary dictionary, int maxFilters, bool readParameters, bool keep) + { + var reader = default(Resolving); + + if (!TryRead(dictionary, maxFilters, ref reader, readParameters, keep, out var reading)) + { + // The steps read whole before the value stand: the chain ends at the element that value was read for. + reading.End = FilterChainEnd.Guard; + reading.EndIndex = reading.Length; + reading.EndValue = null; + } + + return reading; + } + /// Adds to , which grows as a list does. private static void Append(ref T[]? items, ref int count, T item) { @@ -426,12 +449,22 @@ private struct Reading() public FilterParameters Lzw = new(); } - /// Follows each value as does, and never ends the reading. + /// + /// Follows each value as does, and ends the reading at one that leads to an object + /// one of the reader's guards cut, or left out of the index (). + /// private struct Resolving : IFilterValueReader { public readonly bool TryRead(PdfName key, PdfObject? written, bool entry, out PdfObject? value) { value = written?.Resolve(); + + if (written is PdfReference reference && reference.Source is PdfFileReader reader && reader.LeftByGuard(reference)) + { + value = null; + return false; + } + return true; } } diff --git a/src/AdCodicem.Pdf/IO/Filters/FilterChainEnd.cs b/src/AdCodicem.Pdf/IO/Filters/FilterChainEnd.cs index dfa82ee5..e01902d0 100644 --- a/src/AdCodicem.Pdf/IO/Filters/FilterChainEnd.cs +++ b/src/AdCodicem.Pdf/IO/Filters/FilterChainEnd.cs @@ -20,4 +20,11 @@ internal enum FilterChainEnd /// An element past the most filters the reader's guard lets a stream's data pass through. Limit, + + /// + /// A value decoding reads — the /Filter entry or an element of it, /DecodeParms or a value of a step's + /// parameters — that leads to an object one of the reader's guards cut, or left out of the index: what it holds is + /// the guard's, and the step it is read for, and those after it, do not run (#263). + /// + Guard, } diff --git a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs index d91de075..23af747e 100644 --- a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs +++ b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs @@ -33,23 +33,33 @@ public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diag /// /// Bounds what each filter's output may be and how many filters run, and says what reaching either does. public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diagnostics, PdfLimitGuard guard) => - Decode(stream, diagnostics, guard, out _); + Decode(stream, diagnostics, guard, out _, out _); /// - /// Decodes a stream's data under its document's guards, and says whether the most filters the reader lets data pass - /// through ended decoding: what comes back is then the data still encoded by the filters past them, no part of what - /// the stream decodes to, as a reader of its rows or its objects must know. + /// Decodes a stream's data under its document's guards, and says whether one of them ended decoding — the most filters + /// the reader lets data pass through, or a value the filters read that a guard cut or left out of the index: what comes + /// back is then the data still encoded by the filters past it, no part of what the stream decodes to —, and whether one + /// cut the data, as a reader of its rows or its objects must know. /// /// The stream to decode. /// Receives what decoding met; when none are supplied, the stream's document does. - /// Whether ended decoding. + /// + /// Whether ended decoding, or a value the filters read that + /// a guard cut or left out of the index did (#263). + /// + /// + /// Whether the data decoded is cut by a guard: the stream's own data, which the reader read in part, or what a filter + /// stopped at . A guard that loading a value the filters + /// read reaches is that value's, and does not cut the data. + /// /// /// Decoding reaches one of its document's guards, and the document was opened to throw when it does. /// - public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diagnostics, out bool leftEncoded) => - Decode(stream, diagnostics, stream.Data.LimitGuard, out leftEncoded); + public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diagnostics, out bool leftEncoded, out bool cutByGuard) => + Decode(stream, diagnostics, stream.Data.LimitGuard, out leftEncoded, out cutByGuard); - private static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diagnostics, PdfLimitGuard guard, out bool leftEncoded) + private static ReadOnlyMemory Decode( + PdfStream stream, PdfDiagnostics? diagnostics, PdfLimitGuard guard, out bool leftEncoded, out bool cutByGuard) { diagnostics ??= guard.DocumentDiagnostics; @@ -66,7 +76,7 @@ private static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? dia // its own reports in. if (diagnostics is null || !ReferenceEquals(diagnostics, guard.DocumentDiagnostics)) { - return DecodeChain(stream, chain, data, diagnostics, guard, out leftEncoded); + return DecodeChain(stream, chain, data, diagnostics, guard, out leftEncoded, out cutByGuard); } var decodings = diagnostics.Decodings; @@ -74,7 +84,7 @@ private static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? dia try { - return DecodeChain(stream, chain, data, decodings, guard, out leftEncoded); + return DecodeChain(stream, chain, data, decodings, guard, out leftEncoded, out cutByGuard); } finally { @@ -85,7 +95,13 @@ private static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? dia /// Decodes through the filters of , reporting into . private static ReadOnlyMemory DecodeChain( - PdfStream stream, FilterChain chain, ReadOnlyMemory data, PdfDiagnostics? diagnostics, PdfLimitGuard guard, out bool leftEncoded) + PdfStream stream, + FilterChain chain, + ReadOnlyMemory data, + PdfDiagnostics? diagnostics, + PdfLimitGuard guard, + out bool leftEncoded, + out bool cutByGuard) { leftEncoded = false; @@ -128,6 +144,7 @@ private static ReadOnlyMemory DecodeChain( // library does not know: the filters after it would read what it encoded, and each fail on it again. if (failed) { + cutByGuard = cut; return data; } @@ -135,7 +152,11 @@ private static ReadOnlyMemory DecodeChain( } ReportEnd(chain, diagnostics, stream.Data.Position, guard); - leftEncoded = chain.End == FilterChainEnd.Limit; + + // A value a guard cut or left out of the index was reported where the guard was reached: the chain ends there in + // silence. + leftEncoded = chain.End is FilterChainEnd.Limit or FilterChainEnd.Guard; + cutByGuard = cut; return data; } diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 261c7915..6f144071 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -272,6 +272,12 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// private HashSet? _cutWhileChainIsRead; + /// + /// The objects a guard cut that a cross-reference stream's values led to, which the chain could not read and ended at: + /// each is loaded once the chain is read, so that its guard is reached where a load reaches it (#263). + /// + private List? _chainValuesCut; + /// /// The object numbers whose rows a guard left unread in the cross-reference stream just read — past the data it cut, or /// from the row the index had no room for —, as pairs of a first number and a count; null where the section just read @@ -2097,6 +2103,7 @@ private void Initialize() _structure.SizeAsWritten = Trailer.GetRaw(PdfName.Size); _structure.RootAsWritten = Trailer.GetRaw(PdfName.Root); CheckDeferredLengths(); + ReachChainValuesCut(); if (!HasUsableRoot()) { @@ -2188,6 +2195,26 @@ private void CheckDeferredLengths() } } + /// + /// Loads, once the chain is read, each object a guard cut that a cross-reference stream's value led to, which the chain + /// ended at: the guard is reached, and reported, where a load reaches it — and thrown, when the caller asked for it —, + /// as for a /Length read once the chain is (#263). + /// + private void ReachChainValuesCut() + { + if (_chainValuesCut is not { } cut) + { + return; + } + + _chainValuesCut = null; + + foreach (var id in cut) + { + GetObject(id); + } + } + /// /// Gives where the endstream keyword starts that the white space and comments at /// lead to, or itself when none follows them. @@ -3365,12 +3392,6 @@ private static bool SyntaxFaultSince(PdfDiagnostics diagnostics, PdfDiagnosticsM return false; } - /// - /// Determines whether a guard was reached since , its report kept or dropped: a cut taken for - /// the file's damage once the capacity is reached would let an older section's rows stand for what the cut hid. - /// - private static bool LimitSince(PdfDiagnostics diagnostics, PdfDiagnosticsMark mark) => diagnostics.LimitsReached > mark.Limits; - /// Says what a token is, for a message about what stood where something else should have. private static string DescribeToken(PdfToken token) => token.Kind switch { @@ -3548,9 +3569,18 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect return XRefSectionState.Malformed; } - if (UnreadValue(dictionary) is { } unread) + if (UnreadValue(dictionary, out var unreadCut) is { } unread) { + // A value a guard cut is the guard's, as rows a guard cut are: the section counts as cut, the chain ends here, + // and nothing is rebuilt for it (ADR 34). The guard is reached once the chain is read (#263). section.Fault = unread; + + if (unreadCut is { } cut) + { + section.CutByLimit = true; + (_chainValuesCut ??= []).Add(cut); + } + return XRefSectionState.Malformed; } @@ -3632,11 +3662,11 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect while (true) { - var decoded = Filters.PdfFilterPipeline.Decode(xrefStream, _diagnostics, out var leftEncoded); + var decoded = Filters.PdfFilterPipeline.Decode(xrefStream, _diagnostics, out var leftEncoded, out var cut); // Data the guard on filters left encoded holds none of the section's rows: the section is cut before its first. data = leftEncoded ? default : decoded.Span; - decodedWhole = !LimitSince(_diagnostics, decoding) && !xrefStream.Data.CutByGuard; + decodedWhole = !cut && !leftEncoded; // A row longer than the decoded data is one the data does not hold. Not a guard (ADR 34): the data's length is // bounded by MaxDecodedStreamLength, a guard, and a valid section's rows are in its data. Each width is narrowed @@ -3816,22 +3846,24 @@ private static bool DeclaresCrossReferenceStream(PdfObject value) => /// rows are read without — /Type, /Size beside an /Index, an entry no filter reads — leaves the /// section readable; the validator reports how it is written. /// - private string? UnreadValue(PdfDictionary dictionary) + /// The cross-reference stream's dictionary. + /// The object a guard cut that the value which cannot be read leads to, or null when no guard did. + private string? UnreadValue(PdfDictionary dictionary, out PdfObjectId? cut) { - if (Unread(dictionary, PdfName.W, out var widths) is { } fault) + if (Unread(dictionary, PdfName.W, out var widths, out cut) is { } fault) { return fault; } for (var i = 0; widths is PdfArray w && i < Math.Min(3, w.Count); i++) { - if (Follow(PdfName.W, w[i], entry: false, out _) is { } elementFault) + if (Follow(PdfName.W, w[i], entry: false, out _, out cut) is { } elementFault) { return elementFault; } } - if (Unread(dictionary, PdfName.Index, out var index) is { } indexFault) + if (Unread(dictionary, PdfName.Index, out var index, out cut) is { } indexFault) { return indexFault; } @@ -3840,19 +3872,21 @@ private static bool DeclaresCrossReferenceStream(PdfObject value) => { foreach (var element in ranges) { - if (Follow(PdfName.Index, element, entry: false, out _) is { } elementFault) + if (Follow(PdfName.Index, element, entry: false, out _, out cut) is { } elementFault) { return elementFault; } } } - else if ((index is null or PdfNull) && Unread(dictionary, PdfName.Size, out _) is { } sizeFault) + else if ((index is null or PdfNull) && Unread(dictionary, PdfName.Size, out _, out cut) is { } sizeFault) { return sizeFault; } var following = new Following(this); - return Filters.FilterChain.TryFollow(dictionary, _guard.Bound(PdfLimit.FilterChain), ref following) ? null : following.Fault; + var filtersRead = Filters.FilterChain.TryFollow(dictionary, _guard.Bound(PdfLimit.FilterChain), ref following); + cut = following.Cut; + return filtersRead ? null : following.Fault; } /// @@ -3865,10 +3899,14 @@ private struct Following(PdfFileReader reader) : Filters.IFilterValueReader /// Gets what could not be read, once a value could not be. public string? Fault { get; private set; } + /// Gets the object a guard cut that the value which could not be read leads to, if a guard cut one. + public PdfObjectId? Cut { get; private set; } + /// public bool TryRead(PdfName key, PdfObject? written, bool entry, out PdfObject? value) { - Fault = reader.Follow(key, written, entry, out value); + Fault = reader.Follow(key, written, entry, out value, out var cut); + Cut = cut; return Fault is null; } } @@ -3877,8 +3915,8 @@ public bool TryRead(PdfName key, PdfObject? written, bool entry, out PdfObject? /// Follows the value gives in , and says so when a reference it is, /// or leads to, cannot be read. /// - private string? Unread(PdfDictionary dictionary, PdfName key, out PdfObject? value) => - Follow(key, dictionary.GetRaw(key), entry: true, out value); + private string? Unread(PdfDictionary dictionary, PdfName key, out PdfObject? value, out PdfObjectId? cut) => + Follow(key, dictionary.GetRaw(key), entry: true, out value, out cut); /// /// Follows through as many references as does, each read @@ -3889,9 +3927,11 @@ public bool TryRead(PdfName key, PdfObject? written, bool entry, out PdfObject? /// The value as the file wrote it. /// Whether is the entry's own value, rather than an element of it. /// The value it leads to, or null when it leads to none. - private string? Follow(PdfName key, PdfObject? written, bool entry, out PdfObject? value) + /// The object a guard cut that left the value unread, or null when none did. + private string? Follow(PdfName key, PdfObject? written, bool entry, out PdfObject? value, out PdfObjectId? cut) { value = written; + cut = null; for (var link = 0; value is PdfReference reference; link++) { @@ -3903,10 +3943,19 @@ public bool TryRead(PdfName key, PdfObject? written, bool entry, out PdfObject? if (ReadWithoutLoading(reference.Id) is not { } read) { + // An object a section read so far places, which runs past MaxObjectLength: the guard's, not the file's. + var place = "which no section read before it places where it can be read"; + + if (_cutWhileChainIsRead?.Contains(reference.Id) == true) + { + cut = reference.Id; + place = "which one of the reader's limits cut"; + } + value = null; return string.Create( CultureInfo.InvariantCulture, - $"its {FileQuote.Name(key)} {(entry && link == 0 ? "is" : "holds")} the reference {reference.Id.Number} {reference.Id.Generation} R, which no section read before it places where it can be read"); + $"its {FileQuote.Name(key)} {(entry && link == 0 ? "is" : "holds")} the reference {reference.Id.Number} {reference.Id.Generation} R, {place}"); } value = read; @@ -4971,9 +5020,7 @@ internal ObjectStreamHeaderResult ReadObjectStreamHeader( return ObjectStreamHeaderResult.Unreadable; } - var mark = _diagnostics.GetMark(); - var data = Filters.PdfFilterPipeline.Decode(stream, _diagnostics, out var leftEncoded); - var cut = LimitSince(_diagnostics, mark) || stream.Data.CutByGuard; + var data = Filters.PdfFilterPipeline.Decode(stream, _diagnostics, out var leftEncoded, out var cut); // Data the guard on filters left encoded holds none of the stream's objects: what its header says is unknown. if (leftEncoded) @@ -6327,9 +6374,7 @@ private ObjectStreamContents( return null; } - var mark = diagnostics.GetMark(); - var data = Filters.PdfFilterPipeline.Decode(stream, diagnostics, out var leftEncoded); - var cutByGuard = LimitSince(diagnostics, mark) || stream.Data.CutByGuard; + var data = Filters.PdfFilterPipeline.Decode(stream, diagnostics, out var leftEncoded, out var cutByGuard); // Data the guard on filters left encoded holds none of the stream's objects, as data a guard cut before /First. if (leftEncoded) diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 6e07e149..84a5a323 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -135,6 +135,143 @@ public void A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read(b document.Reader.Structure.Sections.Should().OnlyContain(section => section.State == IO.XRef.XRefSectionState.Read); } + /// The entries of object stream 10 that lead to object 20, and what object 20 holds. + private static readonly (string Entries, string Twenty)[] Values = + [ + ("/Filter 20 0 R", "/FlateDecode"), + ("/Filter [20 0 R]", "/FlateDecode"), + ("/Filter /FlateDecode /DecodeParms 20 0 R", "<< /Predictor 1 >>"), + ("/Filter /FlateDecode /DecodeParms [20 0 R]", "<< /Predictor 1 >>"), + ("/Filter /FlateDecode /DecodeParms << /Predictor 20 0 R >>", "1"), + ]; + + public static TheoryData Cuts + { + get + { + var data = new TheoryData(); + + foreach (var (entries, twenty) in Values) + { + foreach (var cut in new[] { "before the value", "inside the value", "member past the data", "left out of the index" }) + { + data.Add(entries, twenty, cut); + } + } + + return data; + } + } + + [Theory] + [MemberData(nameof(Cuts))] + public void An_object_stream_whose_filter_values_a_guard_cut_is_hidden_and_its_members_taken_as_read(string entries, string twenty, string cut) + { + // The catalog is packed in object stream 10, whose /Filter, /DecodeParms or a value of its parameters leads to object + // 20, which a guard cut — before its value, or inside it: "/Flate" of "/FlateDecode" —, left past the data of an + // object stream it decoded, or out of the index. The stream decoded as no filter, or as "/Flate", unsupported, and + // its header read as stray bytes: the catalog was looked for elsewhere, and a sound file judged broken (#263). The + // filters end at the guard's value, the stream is left encoded and hidden, and its members are the guard's. + var (file, options) = PackedCatalog(entries, twenty, cut); + using var document = PdfDocument.Open(file, options); + + document.Reader.Structure.RootCutByGuard.Should().BeTrue(); + document.Diagnostics.Should().NotContain(entry => + entry.Code == PdfDiagnosticCodes.ObjectStreamUnreadable || entry.Code == PdfDiagnosticCodes.FilterUnsupported || + entry.Code == PdfDiagnosticCodes.TrailerRootRecovered); + var report = new PdfValidator().Validate(document); + report.Contains(PdfValidationRuleIds.FileRootInvalid).Should().BeFalse(); + report.Contains(PdfValidationRuleIds.XRefObjectStreamBroken).Should().BeFalse(); + report.Contains(PdfValidationRuleIds.XRefCheckedInPart).Should().BeTrue(); + + using var control = PdfDocument.Open(file); + control.Catalog.Should().NotBeNull(); + control.Diagnostics.Should().BeEmpty(); + } + + [Fact] + public void A_content_stream_whose_filter_a_guard_cut_comes_back_as_the_filters_before_it_decode_it() + { + // The page's content stream decodes through /ASCIIHexDecode, then object 20, which MaxObjectLength cuts before + // "/ASCIIHexDecode": its data comes back as the first filter decodes it, as past MaxFilterChainLength, and nothing is + // said but the guard — once, it decoded as if the chain ended there, in silence (#263). + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, Page), + [4] = file.Object(4, "<< /Filter [/ASCIIHexDecode 20 0 R] /Length 23 >>\nstream\n333032303330323036443E>\nendstream"), + [20] = file.Object(20, new string(' ', 3000) + "/ASCIIHexDecode"), + }; + file.Table(21, rows, "/Size 21 /Root 1 0 R"); + using var document = PdfDocument.Open(file.Bytes, ObjectGuard); + + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + Encoding.ASCII.GetString(stream.Decode().Span).Should().Be("302030206D>"); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + + using var control = PdfDocument.Open(file.Bytes); + Encoding.ASCII.GetString(control.GetObject(new PdfObjectId(4)).AsStream().Required().Decode().Span).Should().Be("0 0 m"); + control.Diagnostics.Should().BeEmpty(); + } + + [Fact] + public void A_guard_a_filter_value_reaches_as_it_loads_is_not_the_stream_s_own_cut() + { + // Object stream 10's /Filter is object 20, read whole from object stream 11, whose own decoding reaches + // MaxDecodedStreamLength on a later member; 10's /First lies past its data. The guard 20's load reached was taken for + // 10's: its fault went unreported, and its catalog was taken as cut, though the validator, reading 20 from the cache, + // judged it broken (#263). The guard is 11's: 10's fault is the file's, reported by both. + var file = new HandWrittenFile(version: "1.5"); + var rows = new Dictionary + { + [2] = (1, file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), 0), + [3] = (1, file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Resources << >> >>"), 0), + [11] = (1, FlateObjectStream(file, 11, [(20, "/FlateDecode"), (21, "(" + new string('x', 300) + ")")]), 0), + [20] = (2, 11, 0), + [21] = (2, 11, 1), + [10] = (1, FlateObjectStream(file, 10, [(1, "<< /Type /Catalog /Pages 2 0 R >>")], "/First 999 ", filter: "20 0 R"), 0), + [1] = (2, 10, 0), + }; + file.CrossReferenceStream(12, 22, rows, "/Root 1 0 R"); + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxDecodedStreamLength = 100 } }; + using var document = PdfDocument.Open(file.Bytes, options); + + document.Reader.Structure.RootCutByGuard.Should().BeFalse(); + document.Diagnostics.Should().Contain(entry => entry.Code == PdfDiagnosticCodes.ObjectStreamUnreadable); + new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefObjectStreamBroken).Should().BeTrue(); + } + + [Theory] + [InlineData("Filter")] + [InlineData("DecodeParms")] + [InlineData("Columns")] + [InlineData("W")] + [InlineData("W element")] + [InlineData("Index")] + [InlineData("Index element")] + [InlineData("Size")] + public void A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s(string key) + { + // Stream 7's /Filter, /DecodeParms, /W or /Index is object 5, which the newer table places and MaxObjectLength cuts + // after its value. The chain said no section placed it, rebuilt the index and judged the section malformed. The + // section is the guard's: the chain ends there, nothing is rebuilt, and the guard is reached once the chain is read + // — thrown at opening when the caller asks for it (#263). + var file = ChainFiles.UnderAnUpdate(key, placed: true, padding: 3000); + using var document = PdfDocument.Open(file, ObjectGuard); + + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.Reader.Structure.Sections[1].CutByLimit.Should().BeTrue(); + document.Reader.Structure.Sections[1].Fault.Should().EndWith("the reference 5 0 R, which one of the reader's limits cut"); + new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().BeFalse(); + + var throwing = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); + throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); + } + /// /// A page whose content stream, object 4, holds , its /Length object 20, which holds /// ; with no endstream unless . @@ -154,6 +291,66 @@ private static byte[] ContentLengthIn(string twenty, string data, bool endStream return file.Bytes; } + /// + /// The catalog, packed in Flate object stream 10, whose dictionary holds , which lead to object + /// 20, holding ; and the reader's options under which a guard cuts object 20 as + /// says. + /// + private static (byte[] File, PdfReaderOptions Options) PackedCatalog(string entries, string twenty, string cut) + { + var file = new HandWrittenFile(version: "1.5"); + var older = -1L; + + if (cut == "left out of the index") + { + older = file.Table(23, new Dictionary { [20] = file.Object(20, twenty) }, "/Size 23"); + } + + var rows = new Dictionary + { + [2] = (1, file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), 0), + [3] = (1, file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Resources << >> >>"), 0), + [1] = (2, 10, 0), + }; + var catalog = "<< /Type /Catalog /Pages 2 0 R >>"; + var data = HostileInputTests.Compress(Encoding.ASCII.GetBytes("1 0 " + catalog)); + rows[10] = (1, file.Object( + 10, + string.Create(CultureInfo.InvariantCulture, $"<< /Type /ObjStm /N 1 /First 4 {entries} /Length {data.Length} >>\nstream\n{Encoding.Latin1.GetString(data)}\nendstream")), + 0); + PdfReaderLimits limits; + + switch (cut) + { + case "before the value": + rows[20] = (1, file.Object(20, new string(' ', 3000) + twenty), 0); + limits = PdfReaderLimits.Default with { MaxObjectLength = 1024 }; + break; + + case "inside the value": + rows[20] = (1, file.Object(20, new string(' ', 1024 - "20 0 obj\n".Length - (twenty.Length / 2)) + twenty + new string(' ', 100)), 0); + limits = PdfReaderLimits.Default with { MaxObjectLength = 1024 }; + break; + + case "member past the data": + rows[11] = (1, FlateObjectStream(file, 11, [(21, "(" + new string('x', 300) + ")"), (20, twenty)]), 0); + rows[21] = (2, 11, 0); + rows[20] = (2, 11, 1); + limits = PdfReaderLimits.Default with { MaxDecodedStreamLength = 100 }; + break; + + default: + limits = PdfReaderLimits.Default with { MaxXRefSectionCount = 1 }; + break; + } + + // Left out of the index, object 20 is numbered by no row of the newest section, which writes a row for every number + // below its /Size. + var size = older < 0 ? 23 : 13; + file.CrossReferenceStream(12, size, rows, "/Root 1 0 R" + (older < 0 ? string.Empty : string.Create(CultureInfo.InvariantCulture, $" /Prev {older}"))); + return (file.Bytes, new PdfReaderOptions { Limits = limits }); + } + /// /// Writes object stream , holding compressed with Flate, or with the /// filter names, its dictionary holding before its own. From 249169e0b55200b0ef19f015581c960999f59c5d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 07:03:35 +0000 Subject: [PATCH 008/146] docs: classify batch 16's bounds in ADR 34 and set them out for the reader's limits ADR 34 gains the amendment of 2026-10-23: MaxStreamLength is a new guard; EndStreamSearchPasses stays an internal constant, its reason rewritten; the bound on the gap before endstream is gone; EndObjLookahead is a first read's size; MaxTrailerLength bounds a cross-reference stream's dictionary alone; a value read through a reference to an object a guard cut or left out of the index is the guard's. The architecture, the threat model and the reader-limits pages say the same. Refs #170, #174, #220, #224, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- ...-is-readable-and-the-readers-guards-are.md | 73 +++++++++++++---- docs/architecture.md | 57 ++++++++----- docs/threat-model.md | 81 +++++++++++-------- docs/website/docs/concepts/reader-limits.md | 10 ++- docs/website/docs/reference/reader-limits.md | 47 +++++++++-- 5 files changed, 189 insertions(+), 79 deletions(-) diff --git a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md index 51dfad9c..08c6c901 100644 --- a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md +++ b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md @@ -13,7 +13,7 @@ saying which bounds a sound file could reach, and follows [8](0008-facade-immuta in making them options on an immutable record. Implemented by `PdfReaderLimits`, `PdfReaderOptions.Limits` and `PdfReaderOptions.ThrowOnLimit`, -`PdfLimitExceededException`, the seven `limit.*` codes of `PdfDiagnosticCodes`, the `readerLimits` field of +`PdfLimitExceededException`, the eight `limit.*` codes of `PdfDiagnosticCodes`, the `readerLimits` field of `tests/corpus/manifest.json`, and `ReaderLimitsTests`. Amended on 2026-09-27, by the maintainer, while M08 was specified: one record bounds every hostile read, the @@ -25,7 +25,8 @@ can make it read, and is an internal constant even where a valid file reaches it of a name (`FileQuote.MaxBytes`), though PDF 2.0 allows a longer one, and at most the first and last four steps of a key path (`RuleText.PathEnds`); the whole value stays in the document. Such a constant's declaration says so, as `No guard (ADR 34): it bounds what a message repeats, not what is read`, in place of the reason no valid file -reaches it. `PdfObjectParser.EndObjLookahead`, which bounds what the reader sees of an object's end, was the first. +reaches it. `PdfObjectParser.EndObjLookahead`, which bounded what the reader saw of an object's end, was the first; the +2026-10-23 amendment made it the size of a first read, and no bound. Amended on 2026-10-14, by the maintainer, while [#164](https://github.com/AdCodicem/AdCodicem.Pdf/issues/164) and [#183](https://github.com/AdCodicem/AdCodicem.Pdf/issues/183) were fixed: a sixth guard, `MaxIndexEntries` @@ -80,6 +81,39 @@ member would start past the data, ends the header as a stray token does. And a h the pairs read. The consequence that raising `MaxDecodedStreamLength` multiplies what the cache of decoded object streams holds until M23 bounds it (T33) was stale: `ObjectStreamBudget` bounds it since 2026-09-28. +Amended on 2026-10-23, by the maintainer, while [#170](https://github.com/AdCodicem/AdCodicem.Pdf/issues/170), +[#174](https://github.com/AdCodicem/AdCodicem.Pdf/issues/174), [#220](https://github.com/AdCodicem/AdCodicem.Pdf/issues/220), +[#224](https://github.com/AdCodicem/AdCodicem.Pdf/issues/224), [#230](https://github.com/AdCodicem/AdCodicem.Pdf/issues/230) +and [#263](https://github.com/AdCodicem/AdCodicem.Pdf/issues/263) were fixed: an eighth guard, `MaxStreamLength` (256 MB, +`limit.stream-length`), bounds what is read of one stream's encoded data at once. The file alone bounded it: decoding +handed back data no filter decodes — none named, an image filter, one the library does not know, Flate over data that is +none — whole, past `MaxDecodedStreamLength` and unreported, and a `/Length` past what an array holds threw an +`OutOfMemoryException` each time the data was read. The data is now kept by nothing and read each time it is asked for, +so the guard bounds each read: past it the data is cut, what the filters lose past the cut is no damage, the stream is +reported once, and its length stays the one the file holds. `MaxDecodedStreamLength` keeps bounding what the filters +produce, so an uncompressed image past the default needs the new guard raised alone. Nor does a stream's data count +toward `MaxObjectLength`: a `/Length` the file cannot hold grew the window an object is parsed through up to that bound, +reading 21.5 MB to find five bytes and reporting `limit.object`; no window grows to find an `endstream` now, and one +search past the window does. What that search reads, with the white space read on after a declared length, is bounded +by `EndStreamSearchPasses`, four times the file for a document's searches together, which stays an internal constant +with its reason rewritten: a valid file is read past the parser's window only by the search for the data of a cross-reference +stream whose `/Length` the chain cannot read, and by white space and comments after a declared length that run past a +first read — about twice the file at most. `MaxEndStreamGap`, the four bytes of white space accepted between a stream's +data and its `endstream`, is removed: more, or a comment, made a right `/Length` a wrong one, and white space and +comments are read on as far as they run, counted with the searches. `EndObjLookahead` is the size of that first read, no +longer a bound. `MaxTrailerLength` bounds a cross-reference stream's dictionary alone: the data of one whose `/Length` the +chain cannot read is searched for its `endstream` in the file rather than through a window the guard bounded — a stream +of 70,000 bytes of rows reads as its twin with a direct `/Length`, with no `limit.trailer`, and `ThrowOnLimit` no longer +throws for it —, and data no `endstream` follows before the next object the sections read so far place, or the end of +the file, makes its section malformed: the file's fault, and no guard's. A value read through a reference to an object a +guard cut or left out of the index — a stream's `/Length`, `/Filter` or `/DecodeParms`, a value of a filter's parameters, +a cross-reference stream's `/W`, `/Index` or `/Size` — is the guard's, not the file's: an integer the cut parted, `1` of +`15`, is only a place to look for `endstream` at, never a declared length; a chain of filters ends there, its data left +encoded; and a cross-reference section whose values lead to an object `MaxObjectLength` cut ends the chain, nothing +rebuilt. A guard that loading a value of a stream's filters reaches is that value's, not a cut of the stream's own data, +whose faults it hid. No bound is added for those values, nor for the next object's header that ends an object's open +containers: no valid object holds `obj` where a value is expected. + ## Context Invariant 4 treats everything read from a file as hostile: no allocation, loop or recursion is sized by a @@ -100,6 +134,7 @@ claims; neither bounded what it was said to, and the 2026-10-14 amendment made t | Bound | Value | What a valid file can do | |---|---|---| | A stream's decoded length, its filters before the last sharing one bound for what they pass along together, a `/Crypt` filter neither charged nor the last | 256 MB | The USGS topographic map in the remote corpus holds one image, 9,600 × 11,410 RGB, that decodes to 313 MB; qpdf reads it whole. A chain of filters whose outputs together pass the bound is valid too, and qpdf reads three thousand of them | +| A stream's encoded data, read at once | 256 MB | ISO 32000 sets no limit on a stream's length, and decoding hands back data no filter decodes as it is: the USGS map's image, stored without a filter, would be 313 MB of data | | An object's length, streams' data aside | 16 MB | A direct array or dictionary of that size: a page-tree node with a million kids | | A classic cross-reference section | 64 MB | About 3.3 million entries; the informative annex C of ISO 32000-1 gives Acrobat's own limit as 8,388,607 objects | | The number of cross-reference sections | 1,024 | A document saved incrementally more than a thousand times | @@ -123,8 +158,8 @@ hands its output out a piece at a time can read the stream (T28). We will read every PDF that is valid under ISO 32000, and treat each bound a valid file can exceed as a guard: active by default, reported when it is reached, and lifted by an option. -- **The options.** `PdfReaderOptions.Limits` is a `PdfReaderLimits`, an immutable record with the seven bounds - of the table: `MaxDecodedStreamLength`, `MaxObjectLength`, `MaxXRefSectionLength`, `MaxXRefSectionCount`, +- **The options.** `PdfReaderOptions.Limits` is a `PdfReaderLimits`, an immutable record with the eight bounds + of the table: `MaxDecodedStreamLength`, `MaxStreamLength`, `MaxObjectLength`, `MaxXRefSectionLength`, `MaxXRefSectionCount`, `MaxTrailerLength`, `MaxIndexEntries` and `MaxFilterChainLength`. `PdfReaderLimits.Default` keeps the values above; `PdfReaderLimits.Unbounded` takes the implementation's maximums — `Array.MaxLength` for a length, `int.MaxValue` for a count. A value of zero or less is refused; a value above the implementation's maximum is @@ -133,26 +168,29 @@ guard: active by default, reported when it is reached, and lifted by an option. configuring anything; one that reads a sound but exceptional file asks for it, by raising one bound or by choosing `Unbounded`. - **Reaching a guard** keeps what fits within it, as the reader does today, and reports it as a warning - under a code of its own — `limit.decoded-stream`, `limit.object`, `limit.xref-section-length`, + under a code of its own — `limit.decoded-stream`, `limit.stream-length`, `limit.object`, `limit.xref-section-length`, `limit.xref-section-count`, `limit.trailer`, `limit.index-entries`, `limit.filter-chain` — whose message names the property that - lifts it. What the parser met at the cut is the reader's, not the file's, and is dropped in favor of the guard. An - object, a section or a trailer is reported once, however often it is read again; a stream, by the first of its filters - that stopped there, each time it is decoded past the bound into diagnostics a caller supplies, and once in its - document's own, however often it is decoded (amended on 2026-10-15); a full index, once a document. The filters after one a guard stopped are given data the reader - cut: that it runs out — a lost tail, a missing checksum, a digit without its pair, a row the predictor cannot finish — - is not reported, and a fault in what was read is. The code `filter.limit-exceeded`, added by T31 and never released, becomes `limit.decoded-stream`. + lifts it. What the parser met at the cut is the reader's, not the file's, and is dropped in favor of the guard; so is a + value that leads to an object a guard cut or left out of the index (amended on 2026-10-23). An object, a section or a + trailer is reported once, however often it is read again; a stream, by the first of its filters that stopped there, + each time it is decoded past the bound into diagnostics a caller supplies, and once in its document's own, however + often it is decoded (amended on 2026-10-15); a stream's data `MaxStreamLength` cut, once for each stream (amended on + 2026-10-23); a full index, once a document. The filters after one a guard stopped, and those of data `MaxStreamLength` + cut, are given data the reader cut: that it runs out — a lost tail, a missing checksum, a digit without its pair, a + row the predictor cannot finish — is not reported, and a fault in what was read is. The code `filter.limit-exceeded`, added by T31 and never released, becomes `limit.decoded-stream`. - **What a guard leaves out of the index is not damage** (amended on 2026-10-14). The objects only a section past - `MaxXRefSectionCount`, rows past `MaxXRefSectionLength` or `MaxDecodedStreamLength`, or numbers past + `MaxXRefSectionCount`, rows past `MaxXRefSectionLength`, `MaxStreamLength` or `MaxDecodedStreamLength`, or numbers past `MaxIndexEntries` index read as null, and the validator treats them as unknown rather than missing, until the property is raised: the index is not rebuilt for them, which turned a sound file into an `xref.rebuilt` error (ADR 45) and could lose what the chain had read. A classic table cut by its length keeps its trailer. The chain ends - at a cut — a table's, a stream's rows, a section or a trailer cut wherever the cut falls —, since an older section, - or its trailer, would stand for what the cut left out; and a section a guard cut is where it was named, not looked for + at a cut — a table's, a stream's rows, a section or a trailer cut wherever the cut falls, a cross-reference stream + whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` leads to an object `MaxObjectLength` cut (amended on + 2026-10-23) —, since an older section, or its trailer, would stand for what the cut left out; and a section a guard cut is where it was named, not looked for near there. A rebuild that damage forces on such a chain keeps the entries the chain read; one the chain cannot do without — a guard that leaves it no section — runs as before. - **Or it throws.** `PdfReaderOptions.ThrowOnLimit`, false by default, turns every guard reached into a `PdfLimitExceededException`, a `PdfException` carrying the code and the name of the property: from - `PdfDocument.Open` when it is reached while opening, from `GetObject` or `Decode` when it is reached later. + `PdfDocument.Open` when it is reached while opening, from `GetObject`, `GetRawBytes` or `Decode` when it is reached later. A rebuild of the index that reaches a guard finishes first and throws after, since a rebuild is never run twice; each later lookup of an object a full index left out throws again. `Open` releases a source it was given to own when it throws. @@ -173,7 +211,8 @@ guard: active by default, reported when it is reached, and lifted by an option. with the defaults. - **Past the implementation's ceiling the rule is not yet met.** A stream that decodes past about 2 GB needs a decode that yields its output a piece at a time; that is T28, for M23, and until then - `limit.decoded-stream` reports it even under `Unbounded`. + `limit.decoded-stream` reports it even under `Unbounded`. A stream's encoded data, read in one piece, has the + same ceiling: `limit.stream-length` reports data past it even under `Unbounded` (amended on 2026-10-23). ## Consequences @@ -184,7 +223,7 @@ guard: active by default, reported when it is reached, and lifted by an option. - The defaults are judgments about what is exceptional, and one is already known to be crossed by a real document: a large-format scan decodes past 256 MB. They can be revised without breaking a caller, who may always set them. -- With `ThrowOnLimit`, any lazy operation — resolving an object, decoding a stream — may throw after `Open` +- With `ThrowOnLimit`, any lazy operation — resolving an object, reading or decoding a stream — may throw after `Open` returned; a caller who sets it must be ready for that wherever the document is used. - `Unbounded` is for documents the application trusts. Raised guards let a file make the reader hold what it asks for, up to the new bounds; the cache of decoded object streams keeps within `ObjectStreamBudget` but always diff --git a/docs/architecture.md b/docs/architecture.md index b53d9c68..ce0a94cb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -94,27 +94,35 @@ separate types, separate severity scales, and vocabularies that never share a co gave is copied before a relocation or a rebuild changes it, so that validation still sees the file's own. 3. **Lazy resolution** — `PdfReference.Resolve()` reads and parses the object on demand. A bounded cache avoids reparsing hot objects (the page tree, shared resources) without ever retaining the whole document. -4. **Streams** — stream data is neither read nor decoded until the caller asks, and then decodes under the - limits of the document it came from; a stream built in memory decodes under the defaults. A stream copied - from one document to another travels **encoded**, with no decompress/recompress cycle. Its declared - length is checked when it is parsed, against the `endstream` that must follow it: in the window the object - is parsed through, or by asking the file for the few bytes after it when the data runs past the window. - A length nothing confirms gives way to the first `endstream` after the data's start: found in the window, - grown up to the object's limit, when the declared length ends inside it or the `/Length` gives none; - searched in the file when a declared length runs past the window — once per stream, no further than the - nearer of the next object the file's index places, as the file wrote it (as the reader rebuilt it when the - document opened, when the file wrote none), and the first object header the file's bytes hold, or the end - of the file, the declared length kept when none lies before them, and the searches of a document reading - a few times the file at most. Neither bound changes as objects are read: what a search keeps depends on - the file's bytes and its index as written, and in the 401 corpus documents measured each stream — each - copy of an object at its offset — takes the same length whatever was read before it. A crafted file can - still make it depend on the number a stream is first read under, on the rebuild's scan for trailers, or, - once the searches of a document have read four times the file, on which streams were searched first - ([#138], M23). A stream the file ends inside runs to the end of the file. Each is a diagnostic. -5. **Guards** — every read sized by the file is bounded. A bound a valid file can exceed — a stream's - decoded length, an object's length, a cross-reference section's length and their number, a trailer's - length — is a `PdfReaderLimits` option, on by default and reported under its own `limit.*` code when - reached; a bound only an invalid file reaches is an internal constant (ADR 34). +4. **Streams** — stream data is neither read nor decoded until the caller asks, nor kept once read + (section 5), and decodes under the limits of the document it came from; a stream built in memory decodes + under the defaults. A stream copied from one document to another travels **encoded**, with no + decompress/recompress cycle. Its declared length is checked when it is parsed, against the `endstream` that + must follow it after white space and comments, however many: in the window the object is parsed through, or + in the file past it, read on as far as the white space and comments run. A length nothing confirms, or a + `/Length` that gives none, goes to one search for the first `endstream` after the data's start ([#174]): in + the bytes the parser holds, then in the file — once per stream, no further than the nearer of the next object + the file's index places, as the file wrote it (as the reader rebuilt it when the document opened, when the + file wrote none), and the first object header the file's bytes hold, or the end of the file, and the + searches of a document reading a few times the file at most. A stream's data never counts toward its + object's length: no window grows to find its `endstream`. Neither bound changes as objects are read: what a + search keeps depends on the file's bytes and its index as written, and in the 401 corpus documents measured + each stream — each copy of an object at its offset — takes the same length whatever was read before it. A + crafted file can still make it depend on the number a stream is first read under, on the rebuild's scan for + trailers, or, once the searches of a document have read four times the file, on which streams were searched + first ([#138], M23). With no `endstream` before the stop, the object's `endobj` ends the data where it + follows the declared length, or where it ends the object before the stop; without one, a declared length + that ends before the stop is kept, and the data otherwise runs to the stop — to the end of the file, for a + stream the file ends inside. Each is a diagnostic. +5. **Guards** — every read sized by the file is bounded. A bound a valid file can exceed — a stream's data + read at once and its decoded length, the filters it passes through, an object's length, a cross-reference + section's length and their number, a trailer's length, the entries of the index — is a `PdfReaderLimits` + option, on by default and reported under its own `limit.*` code when reached; a bound only an invalid file + reaches is an internal constant (ADR 34). What a guard cut is the guard's, never the file's fault: a value + that leads to an object a guard cut or left out of the index — a stream's `/Length`, `/Filter` or + `/DecodeParms`, a cross-reference stream's `/W` — is not judged as one the file gave: the stream's data is + found by its `endstream`, its chain of filters ends there with the data left encoded, and the chain of + cross-reference sections ends at that section. ### 3.2 The write path @@ -163,7 +171,10 @@ goes wrong. No intermediate layer may drop it. ## 5. Memory and CPU strategy - **Memory budget**: consumption follows the complexity of the **page** being processed, never the size of - the document. A ten-thousand-page report must generate in the footprint of a ten-page one. One decoded + the document. A ten-thousand-page report must generate in the footprint of a ten-page one. A stream's + encoded data is kept by nothing: it is read each time it is asked for — lent where it lies by a source held + in memory, read into an array of its length from any other —, at most `PdfReaderLimits.MaxStreamLength` of it + at once, so that streams whose data share a stretch of the file hold no copy of it each ([#170]). One decoded stream is still held whole, up to `PdfReaderLimits.MaxDecodedStreamLength` and at most `Array.MaxLength`, until M23 decodes a piece at a time ([#48]), in native memory if a measurement asks for it (ADR 35). Of a chain of filters, the filters before the last share that bound for what they pass along together, so that decoding a @@ -240,3 +251,5 @@ publication. [#34]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/34 [#48]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/48 [#138]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/138 +[#170]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/170 +[#174]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/174 diff --git a/docs/threat-model.md b/docs/threat-model.md index 4b072606..a7184c37 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -30,7 +30,7 @@ it stands, and each test it names was read. A defense found missing is not writt What a caller entrusts to the library when it opens a file it did not produce: - **The process's memory.** Memory follows the heaviest object the caller touches, not the size of the file - (invariant 2). No allocation is sized by a value from the file without a checked bound (invariant 4). The seven + (invariant 2). No allocation is sized by a value from the file without a checked bound (invariant 4). The eight guards cap what a valid but exceptional file may make the reader hold. - **The process's time, and the calling thread.** The aim is that every call does work in proportion to the bytes it reads, under the guards. Several shapes break it today ([#155], [#168], [#181]). No call can be @@ -97,19 +97,24 @@ programs, image files, content streams, XML, encryption, signatures and the netw | `PdfReaderLimits` | Default | Code | Where it applies | |---|---|---|---| | `MaxDecodedStreamLength` | 256 MB | `limit.decoded-stream` | Every filter's output: the last filter of a chain alone, the filters before it together, for what they pass along (`PdfFilterPipeline`) | -| `MaxObjectLength` | 16 MB | `limit.object` | The window a regular object is parsed through, grown ×8 from 8 KB (`PdfFileReader.TryParseNumberedAt`), and an object stream member's, from its start to the next member's (`ObjectStreamMemberWindow.Of`, `ReachMemberLimit`) | +| `MaxStreamLength` | 256 MB | `limit.stream-length` | One stream's encoded data, read in one piece each time it is asked for, by `GetRawBytes` or a decoding (`FileStreamData.GetBytes`): past it the data is cut, and what lies within it is decoded as data the reader cut, whose lost tail is no damage. Data decoding hands back as it is — no filter, an image filter, one the library does not know, Flate over data that is none — is bounded by it, not by `MaxDecodedStreamLength`. Its ceiling is `Array.MaxLength`, at which a `/Length` past what an array holds ends; the stream's length stays the one the file holds ([#170]) | +| `MaxObjectLength` | 16 MB | `limit.object` | The window a regular object is parsed through, grown ×8 from 8 KB (`PdfFileReader.TryParseNumberedAt`), and an object stream member's, from its start to the next member's (`ObjectStreamMemberWindow.Of`, `ReachMemberLimit`). A stream's data never counts toward it: no window is grown to find an `endstream` ([#174]) | | `MaxXRefSectionLength` | 64 MB | `limit.xref-section-length` | The window a classic cross-reference table is read through | | `MaxXRefSectionCount` | 1,024 | `limit.xref-section-count` | The `/Prev` links the chain follows | -| `MaxTrailerLength` | 64 KB | `limit.trailer` | A trailer the table's window cut, and a cross-reference stream's dictionary | +| `MaxTrailerLength` | 64 KB | `limit.trailer` | A trailer the table's window cut, and a cross-reference stream's dictionary alone, not its data ([#220]) | | `MaxIndexEntries` | 8,388,607 | `limit.index-entries` | The object numbers the index holds, refused ones counted: a row the chain reads, a header a rebuild reads, a member a rebuild expands, and the room a section makes before its rows (`PdfFileReader.AddFromChain`, `Refuse`, `HasRoomInRebuild`, `ReserveIndex`) | | `MaxFilterChainLength` | 64 | `limit.filter-chain` | The filters a stream's data passes through, counted as its `/Filter` array names them, `/Crypt` included: decoding stops before the first past them, and nothing past them is read (`FilterChain`). What comes back is still encoded, so a cross-reference stream or an object stream the guard ends is read as cut before its data (`PdfFilterPipeline.Decode`'s `leftEncoded`) | Reaching a guard keeps what fits and reports its code (`PdfLimitGuard.Reach`): once per position for an object written -in the file, a section or a trailer, once for each member of each definition of an object stream — at that stream's +in the file, a section, a trailer or a stream's data, once for each member of each definition of an object stream — at that stream's data start, which its members share —, once per document for the index, and for a stream decoded past the bound once in its document's diagnostics, each time into diagnostics the caller passes, counted each time either way ([#144]). What a guard leaves out of the index reads as null until it is raised, and the index is not rebuilt for it: a guard's cut is no -damage. When the document was opened with +damage. A value read through a reference to an object a guard cut or left out of the index, through a chain of +references too, is the guard's as well, whatever the cut left of it — null, or a token it parted —, and the reader +judges none of it as the file's. That holds for a stream's `/Length` ([#224]) and the values its filters read +([#263]), and for a cross-reference stream's `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` that +`MaxObjectLength` cut as the chain was read (*Indexing*); it adds no bound. When the document was opened with `ThrowOnLimit`, reaching a guard throws `PdfLimitExceededException` instead. A stream read from a document decodes under that document's guards, however long after opening. `PdfReaderLimits` refuses a bound of zero or less and takes a length past `Array.MaxLength` as that length. Held by `ReaderLimitsTests` throughout; the @@ -239,7 +244,8 @@ Known gaps: |---|---|---|---| | The lexer looping, or reading past its buffer | Every token but the end of input consumes a byte. Look-aheads check the length, and a string that never closes ends at the buffer's end, and is told from a closed one by its token's length | `PropertyTests.The_lexer_terminates_on_any_bytes_and_stays_inside_the_buffer`; `LexerTests.Never_loops_on_a_stray_delimiter`, `Recovers_from_an_unterminated_literal_string`; `ParserTests.Reports_nothing_of_a_construct_closed_at_the_end_of_the_data` | Property, over buffers of about 100 bytes | | Nesting that exhausts the stack | `PdfObjectParser.MaxDepth` (128) and a stack probe at each container. Past either, `SkipContainer` counts brackets without recursion, and `syntax.depth-exceeded` is reported | `ParserTests.Refuses_to_follow_nesting_deeper_than_its_limit`; `HostileInputTests.Survives_deeply_nested_containers_inside_an_object`, `A_file_nesting_objects_deeper_than_the_stack_allows_reads_them_as_null_without_crashing` | No | -| A token or an object of any length, or one that never closes | The window grows ×8 from 8 KB up to `MaxObjectLength`, and the parser is told whether its window ends the data: only then is what it leaves open reported, once, `syntax.truncated-object` where the innermost construct opens, and a window that ends where the file does cuts nothing however long it is. Past the bound, the object is kept as far as it was read and `limit.object` is reported, beside what the parse met before the cut; nothing is reported of the token the cut reaches, nor of what follows once a look-ahead met it (ADR 34). An array or a dictionary an `endobj` finds open ends there, rather than take the objects after it — but for one that lost its own `endobj` too, which takes the next object up to its `endobj` ([#230]). The parses of one object add up to about 1.3 times the last window | `HostileInputTests.Survives_a_string_longer_than_the_largest_window`, `A_string_that_never_closes_takes_the_end_of_the_file_and_is_reported_once_where_it_opens`; `ReaderLimitsTests.An_object_is_read_no_further_than_its_bound`, `An_object_cut_by_its_bound_is_not_reported_as_damage`, `An_object_its_bound_cuts_anywhere_in_its_last_tokens_reports_the_bound_alone`, `What_an_object_its_bound_cuts_met_before_the_cut_is_still_reported`, `A_fault_after_a_stream_whose_length_the_file_cannot_hold_is_still_reported_beside_the_bound`, `An_object_its_bound_cuts_after_more_faults_than_the_diagnostics_hold_still_reports_the_bound`, `A_number_its_bound_cuts_is_no_number_out_of_range`, `An_object_as_long_as_its_bound_that_ends_the_file_is_read_whole`; `WindowEdgeTests.An_anomaly_inside_an_object_longer_than_the_window_is_reported_once`; `PropertyTests.What_a_window_reports_of_its_syntax_the_whole_data_reports_too_in_the_same_order`, `A_buffer_that_ends_at_a_window_s_edge_reports_nothing_of_the_end_of_the_data`; `SyntaxFaultTests.A_construct_the_file_ends_inside_is_reported_where_it_opens`, `A_container_an_endobj_ends_takes_nothing_of_the_objects_after_it`, `A_fault_of_an_object_read_through_a_window_the_file_goes_on_past_is_reported`, `A_rebuild_reports_no_fault_where_its_window_cuts_a_trailer`, `A_trailer_a_source_cut_short_ends_inside_is_reported_as_the_end_of_the_data`; `FileRuleTests.A_trailer_whose_closing_brackets_end_its_table_s_window_is_judged_by_what_they_close` | Growth yes, the bound no; the window's edge by property | +| A token or an object of any length, or one that never closes | The window grows ×8 from 8 KB up to `MaxObjectLength`, and the parser is told whether its window ends the data: only then is what it leaves open reported, once, `syntax.truncated-object` where the innermost construct opens, and a window that ends where the file does cuts nothing however long it is. Past the bound, the object is kept as far as it was read and `limit.object` is reported, beside what the parse met before the cut; nothing is reported of the token the cut reaches, nor of what follows once a look-ahead met it (ADR 34). An array or a dictionary an `endobj` finds open ends there, rather than take the objects after it, and one that lost its own `endobj` too ends at the next object's header (below). The parses of one object add up to about 1.3 times the last window | `HostileInputTests.Survives_a_string_longer_than_the_largest_window`, `A_string_that_never_closes_takes_the_end_of_the_file_and_is_reported_once_where_it_opens`; `ReaderLimitsTests.An_object_is_read_no_further_than_its_bound`, `An_object_cut_by_its_bound_is_not_reported_as_damage`, `An_object_its_bound_cuts_anywhere_in_its_last_tokens_reports_the_bound_alone`, `What_an_object_its_bound_cuts_met_before_the_cut_is_still_reported`, `A_fault_after_a_stream_whose_length_the_file_cannot_hold_is_still_reported_beside_the_bound`, `An_object_its_bound_cuts_after_more_faults_than_the_diagnostics_hold_still_reports_the_bound`, `A_number_its_bound_cuts_is_no_number_out_of_range`, `An_object_as_long_as_its_bound_that_ends_the_file_is_read_whole`; `WindowEdgeTests.An_anomaly_inside_an_object_longer_than_the_window_is_reported_once`; `PropertyTests.What_a_window_reports_of_its_syntax_the_whole_data_reports_too_in_the_same_order`, `A_buffer_that_ends_at_a_window_s_edge_reports_nothing_of_the_end_of_the_data`; `SyntaxFaultTests.A_construct_the_file_ends_inside_is_reported_where_it_opens`, `A_container_an_endobj_ends_takes_nothing_of_the_objects_after_it`, `A_fault_of_an_object_read_through_a_window_the_file_goes_on_past_is_reported`, `A_rebuild_reports_no_fault_where_its_window_cuts_a_trailer`, `A_trailer_a_source_cut_short_ends_inside_is_reported_as_the_end_of_the_data`; `FileRuleTests.A_trailer_whose_closing_brackets_end_its_table_s_window_is_judged_by_what_they_close` | Growth yes, the bound no; the window's edge by property | +| An object that lost both its closing delimiter and its `endobj` taking in the next object, up to that object's `endobj`, its own missing `endobj` unreported; a trailer that lost its `>>` taking the next update's entries | Inside an array or a dictionary, where an element, a key or a value is expected, the next object's header ends every container left open: `N G obj`, digits alone, `N` in 1..`int.MaxValue` and `G` in 0..65,535, white space alone between them, and `obj` a whole keyword the window's edge does not reach — the header the search for a lost `endstream` stops at (`HeaderText`). It is reported once, at the innermost container, as `syntax.truncated-object`, "An object header ended the object inside …", and the object lacks its `endobj`. Containers `SkipContainer` passes over past `MaxDepth` end there too, and a trailer is read so as well. An object stream's member, which no header follows, and a value alone, which is no container, do not apply it: in `4 0 obj 5 0 obj`, object 4 holds the integer 5. Its cost, accepted: a closed container holding text that reads as a header ends there, and its `endobj` is judged missing; none of the 344,361 objects of the corpus holds one ([#230]) | `SyntaxFaultTests.An_object_that_lost_its_closing_delimiter_and_its_endobj_ends_at_the_next_object_s_header`, `The_next_object_s_header_ends_every_container_left_open_and_is_reported_at_the_innermost`, `Text_that_is_no_object_header_ends_no_container`, `A_header_inside_a_closed_container_ends_it_there_and_its_endobj_is_judged_missing`, `An_object_stream_member_holding_a_header_reads_it_as_tokens`, `Nesting_deeper_than_the_reader_follows_ends_at_the_next_object_s_header_too`, `A_trailer_that_lost_its_closing_delimiter_takes_nothing_of_the_update_after_it`; `ParserTests.Ends_the_containers_an_object_left_open_at_the_next_object_s_header_and_leaves_it_to_be_read`, `Takes_an_object_header_a_window_s_edge_reaches_for_none_until_a_larger_window_shows_obj_whole`, `A_header_where_a_key_or_a_value_is_expected_ends_the_dictionary`, `A_value_alone_that_is_a_header_s_number_is_an_integer`; `WindowEdgeTests.An_object_that_lost_its_end_ends_at_the_next_header_wherever_the_window_edge_cuts_it` | Plausibly | | Look-ahead that makes parsing superlinear | Look-ahead is a constant number of tokens, so each token is lexed three times at most | `ParserTests.Distinguishes_a_reference_from_two_integers`, `Backtracks_when_two_integers_are_not_followed_by_R` | Property | | A number past a `long` or a `double` | The number parser never wraps: an integer past a `long` reads as the real nearest to it, and a number past a double's range as an infinity, which the object parser reads as null under `syntax.number-out-of-range`, quoting the number only in a report the diagnostics keep. A real reads as the double nearest to its decimal. The shape — signs, then digits around at most one period — is checked before the framework's parse, which would read `NaN` and `Infinity`; each digit is read twice at most, and nothing is allocated | `LexerTests.Reads_an_integer_too_large_for_a_long_as_a_real`, `Reads_a_number_past_the_largest_double_as_an_infinity_its_reader_can_tell`, `Parsing_a_number_allocates_nothing_however_long_it_is`; `PropertyTests.The_integer_parser_reads_every_long_and_never_wraps_past_one`, `The_real_parser_reads_the_double_nearest_to_the_decimal`; `ParserTests.Reads_a_number_past_the_largest_real_as_null_and_reports_it`, `Reports_no_number_its_double_holds_however_it_rounds`; `HostileInputTests.Reads_a_number_fifteen_million_digits_long_in_time`, `Quotes_no_number_beyond_what_a_real_can_hold_once_the_diagnostics_are_full` | Property | | An object number or a generation overflowing its cast | The parser builds a header only from a number in 1..`int.MaxValue`, and a reference only from one in 0..`int.MaxValue` — object 0 heads the free list and reads as null —, each with a generation in 0..65,535 (`PdfObjectId.MaxNumber` and `MaxGeneration`, with their reasons), checked before the cast. The index and object streams check theirs the same way, before narrowing: see *Indexing* and *Resolving objects and object streams* | `ParserTests.Refuses_an_object_header_whose_number_or_generation_is_out_of_range`, `Makes_no_reference_of_numbers_an_object_identifier_cannot_hold`; `HostileInputTests.Serves_no_object_under_a_number_past_a_long_that_once_wrapped_to_its_number` | No | @@ -255,10 +261,12 @@ Known gaps: - [#163] Nesting past 128 levels, and loads nested past 64, are cut though a valid file can reach both. They become guards. - [#262] A reference a guard cuts between its generation and its `R` reads as two integers. +- Its issue is still to be filed: a name, an integer or a real that a guard's cut parts is kept as the part read — + `/Flate` of `/FlateDecode`, `1` of `15`, `1.0` of `1.5` — by every reader of the object it cut. A stream's `/Length` + and the values its filters read take such an object for the guard's ([#224], [#263]); a caller reading it is told of + the cut, not of the token it parted. - [#36] `PdfString.ToText` misreads PDFDocEncoding, PDF 2.0 UTF-8 strings and UTF-16 language escapes. - [#37] Interned names grow for the life of the process. -- [#230] An object that lost both its closing delimiter and its `endobj` takes the next object up to that object's - `endobj`, and `object.endobj-missing` is not reported for it. ### Indexing: the cross-reference chain, trailers and the rebuild @@ -270,12 +278,13 @@ Known gaps: | A missing section tried at many places | Candidates within 512 bytes either side, nearest first, and 32 attempts at most per document (`MaxRelocationCandidates`); a section a guard cut is where it was named, and is tried nowhere else | `CrossReferenceChainTests.Reads_a_section_that_prev_names_a_few_bytes_off`, `Sections_named_a_few_bytes_off_are_looked_for_at_thirty_two_places_a_document_at_most`; `HostileInputTests.Opening_a_chain_of_hybrid_sections_whose_streams_never_close_reads_a_bounded_amount` | Yes | | A classic table of any length | Its window grows ×4 from 64 KB up to `MaxXRefSectionLength`, and the rows read are kept. A table the guard cuts, or whose rows run past a full index, keeps its trailer, found past the rows left unread by a search a megabyte at a time (`ReadTrailerPastRows`): comments are skipped, and anything else no row holds is the section's fault, reported, the search going on to the `trailer` keyword, so that its `/Encrypt` still refuses the document. The search starts at the start of the line the cut fell in, outside any comment and past the table's own keyword; it reads each megabyte once, every byte of it looked at before the next is read, however many comments or stray bytes it holds, and is not made for a place a section may have been moved to, a guess. The chain ends there, since an older section's rows would stand for the objects the unread rows define. What those rows place reads as null until the guard is raised | `ReaderLimitsTests.A_table_whose_keywords_the_bound_cuts_reports_the_bound`, `A_table_its_length_bound_cuts_keeps_its_trailer_and_the_encryption_it_names`, `A_table_its_length_bound_cuts_ends_the_chain_though_its_trailer_names_an_older_section`, `What_ends_the_rows_a_length_bound_cut_is_judged_as_it_is_where_none_did`, `A_table_a_length_bound_cuts_that_runs_to_the_end_of_the_file_has_no_trailer`, `A_table_its_length_bound_cuts_keeps_the_encryption_its_trailer_names_past_what_no_row_holds`, `A_table_cut_before_its_keyword_ends_keeps_its_trailer`, `Rows_past_a_full_index_are_not_read_and_the_trailer_past_them_is`, `A_cut_in_a_comment_among_the_rows_searches_for_the_trailer_from_the_comment_s_start`, `The_trailer_past_rows_left_unread_is_found_wherever_the_search_s_window_ends`, `A_trailer_past_rows_left_unread_that_its_own_bound_cuts_keeps_what_precedes_the_cut`; `DocumentReaderTests.The_search_for_the_trailer_past_rows_left_unread_reads_the_file_once`; `CrossReferenceChainTests.Reads_an_object_a_length_bound_cut_out_of_its_table_as_null_until_the_guard_is_lifted`, `A_section_found_near_where_prev_names_it_is_not_searched_past_the_rows_a_guard_left_unread`; `WindowEdgeTests.A_classic_table_reads_whole_wherever_its_window_edge_falls_near_its_trailer` | No: seeds stay under 120 KB | | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | -| A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`. The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one` | Yes | -| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread. A value the rows need that cannot be read so makes the section malformed; a `/Length` is read and checked once the chain is, behind a copy of the index as the file wrote it. An object read at a section's offset that is no section leaves nothing of its reading | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount` | No | +| A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one` | Yes | +| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms`, or an element or a parameter of one: the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263]). A `/Length` is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length holds the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …". Rows that spell an `endstream` are told from the data's end by the section's own row for the `/Length`, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. A row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | -| A rebuild run again and again | At most one per document, and none once a validation's first rule has begun: before it, the validator has it run when a later read still could, so that no read of a rule's rebuilds the index halfway through ([#237]). A missing object rebuilds only when the index is known to be incomplete through damage: what a guard left out of it — a section past `MaxXRefSectionCount`, rows past `MaxXRefSectionLength` or `MaxDecodedStreamLength`, numbers past `MaxIndexEntries` — reads as null, unknown to the validator rather than missing, and each later lookup of it throws under `ThrowOnLimit`. A rebuild damage forces on a chain a guard cut keeps the chain's entries the scan does not find. The chain ends wherever the index fills, and wherever a stream's rows are cut, a hybrid file's stream included, so that no older section stands for what the cut rows define; a cut is known by the count of guards reached, which full diagnostics do not drop, not by its report. A catalog or an object stream a guard hid, and the objects such a stream holds, are no reason to rebuild; the validator takes an object stream the chain's index lacks as unknown once a guard stopped the chain, though a rebuild found the stream since | `DocumentReaderTests.A_reference_the_chain_cannot_read_rebuilds_nothing_while_the_chain_is_read`; `CrossReferenceChainTests.Reads_a_reference_to_an_object_the_file_does_not_define_as_null_without_rebuilding`; `ReaderLimitsTests.A_lookup_past_a_full_index_reads_null_and_rebuilds_nothing`, `A_catalog_past_a_guard_cut_reads_as_null_and_rebuilds_nothing`, `A_rebuild_damage_sets_off_keeps_the_entries_a_chain_a_guard_cut_gave_that_the_scan_cannot_find`; `ReadOrderTests.An_entry_whose_object_a_read_would_not_find_has_the_index_rebuilt_before_the_first_rule`; `CorpusValidationTests.No_read_rebuilds_the_index_after_a_validation`; `CrossReferenceRuleTests.An_object_a_guard_left_out_of_the_index_is_no_reference_the_file_lacks`, `An_index_the_entry_bound_filled_is_said_once_to_be_checked_in_part`; `CrossReferenceChainTests.A_full_index_ends_the_chain_at_the_stream_of_a_hybrid_section`, `An_older_section_does_not_stand_for_what_a_cross_reference_stream_s_cut_rows_define`, `A_rebuild_reports_a_full_index_once_though_an_object_stream_lies_before_the_header`; `ReaderLimitsTests.A_catalog_packed_in_an_object_stream_past_a_full_index_reads_as_null_and_rebuilds_nothing`, `A_row_refused_before_a_full_index_is_the_file_s_and_an_object_stream_past_it_stays_unknown`; `DiagnosticsTests.A_guard_reached_is_counted_though_the_capacity_drops_its_report_and_follows_it_when_moved_or_rolled_back` | Yes | +| A rebuild run again and again | At most one per document, and none once a validation's first rule has begun: before it, the validator has it run when a later read still could, so that no read of a rule's rebuilds the index halfway through ([#237]). A missing object rebuilds only when the index is known to be incomplete through damage: what a guard left out of it — a section past `MaxXRefSectionCount`, rows past `MaxXRefSectionLength`, `MaxStreamLength` or `MaxDecodedStreamLength`, a section whose rows need a value `MaxObjectLength` cut ([#263]), numbers past `MaxIndexEntries` — reads as null, unknown to the validator rather than missing, and each later lookup of it throws under `ThrowOnLimit`. A rebuild damage forces on a chain a guard cut keeps the chain's entries the scan does not find. The chain ends wherever the index fills, and wherever a stream's rows are cut, a hybrid file's stream included, so that no older section stands for what the cut rows define; a cut is known by the count of guards reached, which full diagnostics do not drop, not by its report. A catalog or an object stream a guard hid, and the objects such a stream holds, are no reason to rebuild; the validator takes an object stream the chain's index lacks as unknown once a guard stopped the chain, though a rebuild found the stream since | `DocumentReaderTests.A_reference_the_chain_cannot_read_rebuilds_nothing_while_the_chain_is_read`; `CrossReferenceChainTests.Reads_a_reference_to_an_object_the_file_does_not_define_as_null_without_rebuilding`; `ReaderLimitsTests.A_lookup_past_a_full_index_reads_null_and_rebuilds_nothing`, `A_catalog_past_a_guard_cut_reads_as_null_and_rebuilds_nothing`, `A_rebuild_damage_sets_off_keeps_the_entries_a_chain_a_guard_cut_gave_that_the_scan_cannot_find`; `ReadOrderTests.An_entry_whose_object_a_read_would_not_find_has_the_index_rebuilt_before_the_first_rule`; `CorpusValidationTests.No_read_rebuilds_the_index_after_a_validation`; `CrossReferenceRuleTests.An_object_a_guard_left_out_of_the_index_is_no_reference_the_file_lacks`, `An_index_the_entry_bound_filled_is_said_once_to_be_checked_in_part`; `CrossReferenceChainTests.A_full_index_ends_the_chain_at_the_stream_of_a_hybrid_section`, `An_older_section_does_not_stand_for_what_a_cross_reference_stream_s_cut_rows_define`, `A_rebuild_reports_a_full_index_once_though_an_object_stream_lies_before_the_header`; `ReaderLimitsTests.A_catalog_packed_in_an_object_stream_past_a_full_index_reads_as_null_and_rebuilds_nothing`, `A_row_refused_before_a_full_index_is_the_file_s_and_an_object_stream_past_it_stays_unknown`; `DiagnosticsTests.A_guard_reached_is_counted_though_the_capacity_drops_its_report_and_follows_it_when_moved_or_rolled_back` | Yes | | The rebuild's scan | 1 MB pieces overlapping by 64 bytes, each piece after the first searched from where the one before could no longer hold a whole `obj`, so that a header is read once. A header is read backwards, each run of digits judged by its value however many zeros lead it, as the parser judges it: a number in 1..`int.MaxValue` and a generation in 0..65,535, which the entry records; the search near an entry's offset, a load's, settling's and the validator's probe's, reads a header so too. The definitions it finds again of a number are counted, with the first ten numbers, in a fixed-size record, and reported once as `object.redefined` | `DocumentReaderTests.Rebuilds_an_index_whose_objects_lie_past_the_first_megabyte`, `A_rebuild_ignores_object_headers_numbered_zero_or_beyond_what_an_object_number_holds`, `A_rebuild_finds_a_header_whose_number_and_generation_zeros_lead`, `A_rebuilt_entry_records_the_generation_its_header_gives`, `Counts_two_definitions_the_scan_finds_in_the_overlap_of_two_windows_as_one_redefinition`, `Reads_a_header_that_straddles_the_start_of_a_scan_window_under_its_own_number`; `HostileInputTests.Survives_a_file_made_only_of_object_headers`; `CrossReferenceRuleTests.An_entry_at_a_header_the_parser_refuses_is_broken` | Partly | | A catalog a guard cut — its `/Type` or its `/Pages` past the cut —, which leads to no catalog as read | The object `/Root` names, through a chain of references too, is taken for the catalog as read — one an object stream's cut leaves unread, past the cut data or in a stream the guard made unreadable, as one it cut ([#259]), and one written in a trailer `MaxTrailerLength` cut —, at opening and after any rebuild: no search among the indexed objects, which could take a catalog an update superseded, and no rebuild, which would read it cut again; `FileStructure.RootCutByGuard` records it, and `file.root-invalid` does not judge it ([#175]). A rebuild after a search reads the file's own `/Root` again, not the search's guess, and a guard it reaches there is held, as any in a rebuild. A rebuild keeps in its index the members the chain's index places in an object stream a guard hides — the scan finds no header of theirs —, which stay unread for the guard's reasons and throw again when opened to throw; a stream refused for a fault of its dictionary, read whole, hides none, whatever a limit cut of its data | `ReaderLimitsTests.A_catalog_a_guard_cut_is_taken_as_read_and_no_other_is_looked_for_even_by_a_later_rebuild`, `A_catalog_an_object_stream_s_cut_leaves_unread_stays_taken_as_read_through_a_rebuild`, `A_catalog_written_in_a_trailer_a_guard_cut_is_taken_as_read`, `A_catalog_written_in_a_trailer_a_guard_cut_stays_taken_as_read_through_a_rebuild`, `A_member_an_object_stream_s_cut_leaves_unread_throws_again_after_a_rebuild`, `An_object_stream_refused_for_its_own_fault_hides_no_member_where_a_limit_also_cut_its_data`, `A_rebuild_after_a_search_reads_the_file_s_own_root_again`, `A_rebuild_reaching_a_guard_on_a_catalog_a_search_took_still_looks_for_another_when_opened_to_throw`, `A_root_that_leads_through_a_chain_to_a_catalog_a_guard_cut_is_taken_as_read`, `A_catalog_an_object_stream_s_cut_leaves_unread_is_taken_as_read`; `FileRuleTests.A_catalog_a_limit_cut_inside_its_type_is_not_judged`, `A_catalog_a_limit_cut_before_its_type_is_taken_as_read_rather_than_an_older_one`, `A_catalog_that_is_none_is_still_judged_beside_one_a_limit_cut_elsewhere` | No | | A guard reached in the middle of a rebuild | The guard's exception is held, and thrown once the index is whole | `ReaderLimitsTests.A_rebuild_that_reaches_a_guard_finishes_the_index_before_it_throws`, `A_rebuild_that_reaches_a_guard_looking_for_the_catalog_still_finds_it`; `ReadOrderTests.A_guard_the_rebuild_before_the_first_rule_reaches_throws_in_the_first_validation_only` | No | @@ -289,8 +298,6 @@ Known gaps: bucket, and every lookup walks it. - [#171] A rebuild takes headers cut at a chunk's edge, or lying in stream data or strings, and serves another object under their number. -- [#220] A cross-reference stream whose `/Length` only it indexes is found by an `endstream` search through a window - `MaxTrailerLength` bounds, a guard named for a dictionary; rows that spell `endstream` end its data early. - [#221] A cross-reference stream row of a reserved type is skipped, letting an older section's row stand where ISO 32000-1 reads the object as null. - [#156] An `/XRefStm` naming a section already read, or a classic table, leaves no record and no finding. @@ -311,7 +318,7 @@ Known gaps: | Threat | Defense | Held by | Fuzzed | |---|---|---|---| | An object whose load needs itself, or loads chained without end | `_loading` turns a load of an object being loaded into null. `PdfFileReader.MaxNestedLoads` (64) and a stack probe bound the chain, reported once as `syntax.depth-exceeded` | `HostileInputTests.Survives_an_object_whose_length_refers_to_itself`, `Survives_a_chain_of_lengths_longer_than_the_stack_is_deep`, `Reports_chains_that_run_too_deep_once` | The load path, not the bound | -| Decodes nested through filter parameters, each holding its output while the next decodes | A stream resolves its filters, its parameters and the values they hold before it reads any of its data, and holds them as plain values: only what a step reads — LZW's `/EarlyChange`, a Flate or LZW step's predictor values —, nothing past an image filter, an unknown one or an element that names none. A chain of loads holds what `ObjectStreamBudget` keeps, the object stream decoded last, and one decoding, whatever its depth ([#165]), besides the raw data of each object stream on it, which the object cache keeps until it lets the stream go ([#170]) | `FilterTests.Resolves_every_value_a_step_reads_before_any_step_reads_the_data`, `Resolves_nothing_a_step_does_not_read`, `Reads_a_stream_s_data_from_the_file_once_every_value_its_filters_read_is_loaded`; `HostileInputMemoryTests.A_chain_of_object_streams_each_needing_the_next_to_decode_holds_one_level_s_output_at_once` (64 levels of 4 MB, held to 32 MB plus three bounds, live at once) | No | +| Decodes nested through filter parameters, each holding its output while the next decodes | A stream resolves its filters, its parameters and the values they hold before it reads any of its data, and holds them as plain values: only what a step reads — LZW's `/EarlyChange`, a Flate or LZW step's predictor values —, nothing past an image filter, an unknown one or an element that names none. A chain of loads holds what `ObjectStreamBudget` keeps, the object stream decoded last, and one decoding, whatever its depth ([#165]); no object stream on it keeps the raw data it read ([#170]) | `FilterTests.Resolves_every_value_a_step_reads_before_any_step_reads_the_data`, `Resolves_nothing_a_step_does_not_read`, `Reads_a_stream_s_data_from_the_file_once_every_value_its_filters_read_is_loaded`; `HostileInputMemoryTests.A_chain_of_object_streams_each_needing_the_next_to_decode_holds_one_level_s_output_at_once` (64 levels of 4 MB, held to 32 MB plus three bounds, live at once) | No | | References to references that loop | `PdfReference.Resolve` gives up after 32 fetches and reads null, with no report ([#173]) | `ObjectModelTests.A_chain_of_references_that_loops_resolves_to_null` | No | | An object stream that needs an object it holds, or two that need each other | While an object stream is decoded, an object it holds reads as null and is reported once as `stream.self-reference`, at the stream's entry, or nowhere when the entry lies outside the file; the null is remembered, so a later decode reads the same | `HostileInputTests.An_object_stream_that_needs_an_object_it_holds_reads_the_same_after_the_budget_lets_it_go`, `Two_object_streams_that_need_each_other_read_the_same_after_the_budget_lets_them_go`; `HeaderOffsetTests.An_object_stream_that_needs_itself_found_near_a_row_past_the_end_is_reported_nowhere` | Object-stream seeds, not the cycle | | An offset table that lies | `/First` is checked against the decoded data, and the stream against its `/Type`, which must be `/ObjStm` ([#274]). The header is read as far as it goes, whatever `/N` claims, up to its first pair that is not two integers, that gives a number outside 1..`int.MaxValue` or an offset outside 0..`int.MaxValue`, whose offset is not past the one before it, or whose member would start at or past the end of the decoded data unless a guard cut it ([#259]) — the reader, the rebuild and the validator alike ([#202]) —: the pairs before it are kept, the fault is reported once as `object-stream.unreadable`, and the validator counts the objects the index places in the stream that the header does not list before it, and lost — none, and no finding, when it lists them all ([#275]). The header's tables grow as its pairs are read, never past the quarter of `/First`'s bytes a header holds at most, so that a `/N` no header holds allocates nothing ([#160]). A wrong index falls back on a lookup by number, sorted and built once (`ObjectStreamPlaces`), reported once for each object and definition of its stream as `object-stream.member-moved`: the object is read at the first place the header lists it. The validator's probe finds each entry's object through the same lookup, and the members written as a stream by a search, so that judging a stream costs what its members do | `ReaderTimingTests.An_object_stream_whose_every_index_is_wrong_is_read_in_linear_time`; `DocumentReaderTests.Reads_an_object_where_its_stream_s_header_lists_it_rather_than_where_the_index_says`, `Finds_an_object_its_entry_misplaces_at_the_first_place_its_object_stream_lists_it`, `Reports_an_object_stream_whose_dictionary_or_header_cannot_be_believed_where_its_data_starts`, `Reads_no_member_an_object_stream_header_numbers_as_no_object_can_be`, `Reads_no_member_an_object_stream_header_places_where_no_member_can_start`, `Ends_an_object_stream_header_at_a_member_that_starts_before_the_one_before_it_ends_or_past_the_data`, `Reads_none_of_an_object_stream_whose_first_member_starts_past_its_data`, `Reads_none_of_the_members_of_a_stream_that_is_not_of_type_object_stream`, `Reads_an_object_stream_header_that_lists_the_largest_number_a_member_can_have`, `Indexes_no_member_a_rebuild_finds_listed_under_a_number_no_object_can_have`, `Reports_an_object_at_another_index_of_its_stream_once_however_often_it_is_parsed`; `HostileInputTests.An_object_stream_that_claims_more_objects_than_it_could_hold_serves_those_it_lists_and_allocates_nothing_for_the_rest`, `An_object_stream_whose_N_its_blank_header_cannot_hold_allocates_no_table_for_it`, `An_object_stream_that_lists_one_offset_over_and_over_keeps_one_member`; `CrossReferenceRuleTests.An_object_stream_whose_header_lists_something_other_than_objects_cannot_be_read`, `An_object_stream_whose_header_breaks_after_an_object_says_which_objects_are_lost`, `An_object_stream_whose_first_member_starts_past_its_data_cannot_be_read`, `An_object_stream_that_cannot_serve_its_objects_is_reported_once`, `An_object_stream_that_lists_every_object_its_index_places_before_its_header_breaks_loses_none_and_is_no_finding`, `An_object_stream_broken_after_an_object_the_index_misplaces_shifts_that_object_and_loses_the_other`; `HostileInputTests.An_object_stream_header_s_tables_start_no_larger_than_its_bytes_can_hold`, `An_object_stream_header_s_tables_grow_no_larger_than_its_bytes_can_hold`; `ReadOrderTests.A_member_the_definition_a_rebuild_serves_lists_elsewhere_is_reported_moved_whatever_was_read_before`; `ReaderTimingTests.The_validator_judges_an_object_stream_s_members_in_linear_time` | Yes | @@ -319,7 +326,7 @@ Known gaps: | A stream written inside an object stream, which ISO 32000 forbids | The parser stops at its `stream` keyword: the object reads as null, nothing of it is decoded nor kept, and it is reported once for each definition of its stream and number as `object-stream.member-stream`; the validator reports the entry that places it there as `xref.entry-broken`, naming the index its header gives when the entry gives another, and the entry of the index a rebuild made where the file's own places nothing — or, where a guard stopped the chain, the part of it the reader read ([#161]) | `DocumentReaderTests.Reads_an_object_written_as_a_stream_inside_an_object_stream_as_null_and_reports_it_once`; `HostileInputTests.A_stream_inside_an_object_stream_is_not_decoded_whatever_it_would_decode_to`; `HostileInputMemoryTests.Object_streams_each_holding_a_stream_keep_no_more_than_their_budget_once_their_members_are_read`; `StreamLengthTests.A_stream_inside_an_object_stream_has_no_length_read_nor_reported_however_often_it_is_read`; `CrossReferenceRuleTests.An_object_written_as_a_stream_inside_an_object_stream_is_broken`, `An_object_whose_bytes_merely_hold_stream_inside_an_object_stream_is_sound`, `An_object_written_as_a_stream_that_its_entry_misplaces_is_broken_where_its_header_lists_it`, `An_object_written_as_a_stream_that_only_a_rebuilt_index_places_is_broken`, `An_object_written_as_a_stream_that_the_chain_and_a_rebuilt_index_both_place_is_its_entry_s_finding_alone`; `ReadOrderTests.A_member_fault_of_the_definition_a_rebuild_serves_is_reported_whatever_was_read_before` | Object-stream seeds, not a stream member | | Many large object streams held at once | `ObjectStreamBudget` (32 MB, with its reason), which weighs each stream's decoded data with its header's tables and lookup ([#160]): the oldest are let go and decoded again on demand, and nothing a member gave keeps its stream's data alive past them ([#161]) | `HostileInputTests.Object_streams_are_kept_decoded_within_a_budget_and_decoded_again_when_needed`, `The_object_stream_budget_weighs_the_header_tables_with_the_data`; `HostileInputMemoryTests.Disposing_a_document_lets_go_of_the_object_streams_it_decoded_though_a_reference_it_gave_is_held` | No | | A stream decoded again and again, each decoding reporting into its document | Its document's diagnostics hold what decoding a stream met once — a filter's fault, a guard reached on its data —: a decoding reports apart, and a report the document holds already, the whole of it alike, is not moved in, so that it takes no room there; two filters of one decoding that report alike are each held. An object stream decoded again after the budget let it go, by a rebuild, by the validator or by a caller adds nothing it holds already. The guard is still counted, so that what it cut is never taken for damage, and a document opened to throw throws each time, what the decoding met before moved in all the same ([#144]) | `HostileInputTests.A_fault_of_an_object_stream_is_held_once_by_its_document_however_often_the_stream_is_decoded`, `A_guard_an_object_stream_reaches_is_held_once_by_its_document_and_counted_each_time`, `A_document_holds_alike_reports_of_two_streams_and_of_two_filters_of_one_stream_each_once_a_decoding`, `What_a_decoding_met_before_a_guard_threw_is_held_once_though_each_decoding_throws`, `A_report_its_document_holds_takes_no_room_when_a_stream_is_decoded_again_into_full_diagnostics`, `A_report_the_capacity_leaves_out_of_one_decoding_is_counted_by_its_document`, `A_report_full_diagnostics_left_out_is_counted_again_each_time_its_stream_is_decoded`; `DiagnosticsTests.A_copy_of_a_report_the_capacity_left_out_is_counted_again_each_time_a_decoding_meets_it`, `A_decoding_moves_in_only_what_no_decoding_moved_in_before_two_alike_of_its_own_included`, `A_report_a_decoding_moved_in_before_takes_no_room_once_the_diagnostics_are_full_and_a_guard_stays_counted`; `CorpusReadingTests.A_document_holds_each_report_once_however_often_its_streams_are_decoded`, `A_damaged_object_stream_s_fault_is_held_once_after_a_full_read_and_a_validation` | No | -| Parsed objects piling up | The object cache is FIFO, bounded by count (`ObjectCacheCapacity`, 8,192), not by weight ([#37], [#170]), and keyed by number, so an object named under many generations is parsed once | `HostileInputTests.An_object_referenced_under_many_generations_is_read_once`; `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go` | No | +| Parsed objects piling up | The object cache is FIFO, bounded by count (`ObjectCacheCapacity`, 8,192), not by weight ([#37]) — a stream it holds keeps none of its data ([#170]) —, and keyed by number, so an object named under many generations is parsed once | `HostileInputTests.An_object_referenced_under_many_generations_is_read_once`; `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go` | No | | An object that is not where its entry says | The entry is checked against the file, before the header's offset is added, a header is looked for near it — within 512 bytes, though a window clamped at the file's start reaches further ([#197]); none for an offset more than 512 bytes before the file's start, nor past what a `long` holds —, and then the index is rebuilt once: three counted attempts. This answers a stack overflow the fuzzing found in M01. A moved entry records the generation of the header found, not that of the reference that asked, and that the reader found it | `DocumentReaderTests.Finds_objects_whose_recorded_offsets_are_wrong`, `Finds_an_object_whose_entry_points_past_the_end_of_the_file`, `An_object_a_rebuild_does_not_find_either_is_null`, `A_relocated_entry_records_the_generation_its_header_gives_whatever_reference_found_it`; `HeaderOffsetTests.An_entry_past_what_a_long_holds_is_not_looked_for_at_the_file_s_start_whatever_precedes_the_header`, `An_entry_far_before_the_file_s_start_is_not_looked_for_at_it`, `An_object_found_near_its_entry_before_the_header_is_placed_where_it_was_found` | Yes | | A reference to what the index lacks, forcing a rebuild each time | A rebuild runs only when the index is known to be incomplete through damage, and once; what a guard left out reads as null with none | `CrossReferenceChainTests.Reads_a_reference_to_an_object_the_file_does_not_define_as_null_without_rebuilding`; `ReaderLimitsTests.A_lookup_past_a_full_index_reads_null_and_rebuilds_nothing` | Yes | @@ -337,26 +344,32 @@ Known gaps: | Threat | Defense | Held by | Fuzzed | |---|---|---|---| -| A `/Length` that is negative, past `int.MaxValue`, not an integer or unresolvable | Only 0..`int.MaxValue` is taken as a length. Anything else sends the reader to the first `endstream` in the window | `StreamLengthTests.A_length_that_cannot_be_taken_is_reported_as_the_file_wrote_it` | Partly | +| A `/Length` that is negative, past `int.MaxValue`, not an integer or unresolvable | Only 0..`int.MaxValue` is taken as a length. Anything else sends the reader to the one search for the stream's first `endstream` (below) | `StreamLengthTests.A_length_that_cannot_be_taken_is_reported_as_the_file_wrote_it` | Partly | +| A `/Length` naming an object a guard cut or left out of the index, judged by what the cut left of it: null, reported as no length, or the `1` it parted of `15`, reported as a length the data contradicts | Keyed on the object the reference leads to, through a chain of references too, never on the value read (`StreamLengthForm.ObjectCut`, `ObjectUnindexed`): an integer read before the cut is only a place to look for `endstream` at, never kept as a length, and the data is otherwise found by its `endstream`. Nothing is said of the length, whose fault is the guard's, reported where it was reached; a missing `endstream` is still the file's, and reported: "The stream's /Length names object N G, which one of the reader's limits cut, and no endstream follows its data …". A cross-reference stream's `/Length` so cut is left alike once the chain is read ([#224]) | `GuardCutValueTests.A_length_a_guard_cut_is_not_judged_and_the_data_is_found_by_its_endstream`, `A_length_a_guard_cut_leaves_a_missing_endstream_the_file_s_fault`, `A_length_an_object_stream_s_guard_left_unread_is_not_judged`, `A_length_whose_reference_leads_to_an_object_a_guard_left_out_of_the_index_is_not_judged`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read` | No | | An indirect `/Length` leading back to its stream, or chained | The loading set and `MaxNestedLoads`, as above | `HostileInputTests.Survives_an_object_whose_length_refers_to_itself`, `Survives_a_chain_of_lengths_longer_than_the_stack_is_deep` | No | -| A declared length past the file | It is checked against the file before the file is asked, and the length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | -| Confirming a length by reading past the buffer or the file | The look-ahead after the data is a fixed stack buffer of 13 bytes, or 64 past the window, refused outside the file and clamped to it | `WindowEdgeTests.A_stream_the_file_cuts_short_is_reported_without_asking_the_source_past_its_end`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more` | Property | -| A search for a lost `endstream` running into other objects | It stops at the next object the index places, the first object header the bytes hold, or the end of the file. When none of these comes first, the declared length is kept and reported | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_keeps_its_declared_length`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened` | Plausibly | -| Many searches over one stretch of the file | The searches past the window read at most four times the file in a document (`EndStreamSearchPasses`, with its reason). Past that, the declared length is kept unsearched | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over` | No | -| One search reading without end | Reads of at most 64 KB, each strictly after the last, ending at the bound, at the end of the file, at a header, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | -| The same search made at every new parse | The result is kept by data start for the life of the document | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go` | — | +| A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | +| A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found: no window is grown to find an `endstream`, the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_data_start_a_limit_cuts_records_nothing_of_its_length`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows` | No | +| Confirming a length by reading past the buffer or the file, or through white space without end | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. The bytes the parser holds decide when they can. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none` | Property | +| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space between it and the stop, ends the data, its end-of-line left out, `stream.truncated`. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more` | Plausibly | +| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, and by white space after a declared length that runs past a first read, about twice the file at most; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | +| One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | +| The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go` | — | +| Encoded data of any length: a `/Length` of a few digits claiming gigabytes, one past what an array holds, which would exhaust memory at each read, or data decoding hands back as it is, which `MaxDecodedStreamLength` does not bound | `MaxStreamLength` (*The guards*): what is read of one stream's data at once stops at it, what lies past it is cut by a guard, reported once for the stream as `limit.stream-length`, and the filters decode what was read as data the reader cut ([#170]) | `ReaderLimitsTests.A_stream_cut_at_MaxStreamLength_decodes_what_lies_within_it_as_data_the_reader_cut_and_is_reported_once`, `Data_decoding_hands_back_as_it_is_is_bounded_by_MaxStreamLength_and_not_by_MaxDecodedStreamLength`, `A_length_past_what_an_array_holds_ends_at_the_guard_and_never_in_an_exhausted_memory` | No | +| A stream's raw data held for as long as the stream is, and streams whose data share a stretch of the file each holding a copy of it: two hundred sharing a megabyte, 200 MB | Nothing keeps a stream's encoded data (`FileStreamData`): it is read each time it is asked for, by `GetRawBytes` or a decoding. A source in memory lends it where it lies, with no copy, and any other reads it into an array of its length. A stream keeps its place and length, not its bytes ([#170]) | `DocumentReaderTests.A_stream_s_data_is_read_from_the_file_each_time_it_is_asked_for_and_kept_by_nothing`; `HostileInputMemoryTests.Streams_whose_data_share_a_stretch_of_the_file_hold_none_of_it_once_read`, `Validating_object_streams_whose_data_share_a_stretch_of_the_file_holds_no_copy_of_their_data`; `HostileInputTests.Reading_streams_whose_data_share_a_stretch_of_a_file_held_in_memory_allocates_none_of_their_data` | No | | A stream decoded long after opening escaping its document's guards | The stream's data carries its document's guard and diagnostics | `ReaderLimitsTests.A_stream_decoded_without_diagnostics_reports_the_bound_to_its_document` | Yes | Known gaps: -- [#170] Raw data is bounded only by the file. Every cached stream keeps its own copy, overlapping streams multiply - it, and data passed through undecoded escapes `MaxDecodedStreamLength`. -- [#174] A length is misjudged when more than four white-space bytes precede `endstream`, when the file cannot hold - the `/Length`, or when the search inside the window crosses into the next object. +- Its issue is still to be filed: streams whose data share a stretch of the file each read that stretch again from a + source not held in memory, whenever their data is asked for. What overlapping streams cost is time rather than + memory, bounded for each read by `MaxStreamLength` alone; a check for overlapping data is proposed. - [#138] A stream searched for its `endstream` can keep a length that depends on the object read first. -- [#53] Encoded data past about 2 GB cannot be represented, and a length just under `int.MaxValue` throws - `OutOfMemoryException`. +- [#53] Encoded data past about 2 GB cannot be represented: a `/Length` past what an array holds is cut at + `MaxStreamLength`, whose ceiling is `Array.MaxLength`. - [#48] Data that decodes past about 2 GB cannot be read whole. +- Accepted, with no issue: the last object of a classic file, whose stream lost its `endstream` and declares no length + within the file the reader can take, takes as its data everything to the end of the file, its cross-reference table + and trailer included: neither `xref` nor `trailer` stops the search ([#174]). ### Decoding @@ -369,14 +382,15 @@ its zlib patched is keeping part of the reader patched. | A decompression bomb | Every decoder writes through `PdfBoundedOutput`, or a buffer capped like it, at `MaxDecodedStreamLength` | `FilterTests.Keeps_exactly_what_the_bound_allows_and_reports_only_a_bound_it_met`; `HostileInputTests.Decodes_a_run_length_stream_no_further_than_the_bound` | Not at the bound | | A `/Filter` array of any length, each filter decoding to the bound | The filters before the last share one `MaxDecodedStreamLength` for what they pass along together, and the last has its own: a chain holds and fills two bounds at most, whatever its length. A `/Crypt` filter, which passes its data along as it is, is neither charged nor the last. A filter that decodes nothing ends the chain, so that a faulty Flate stream is read again once, not once an entry ([#166]) | `FilterTests.Shares_one_bound_among_the_filters_before_the_last`, `Bounds_a_filter_in_the_middle_of_a_chain_as_well_as_the_last_and_reports_the_first_that_stopped`, `Ends_a_chain_at_a_flate_step_that_decodes_nothing_and_returns_what_the_filters_before_it_decoded`, `A_crypt_filter_is_neither_charged_to_the_bound_the_filters_share_nor_the_last`; `HostileInputTests.Opens_a_cross_reference_stream_whose_run_length_filters_multiply_its_data_at_a_cost_its_number_of_filters_leaves_alone`, `Decodes_a_long_chain_of_sound_flate_layers_within_twice_the_bound`, `Decodes_data_no_flate_reading_can_decode_once_however_many_flate_filters_name_it`; `FilterDamageTests.A_chain_of_filters_decodes_to_its_data_or_to_the_start_of_it_with_the_guard_alone_reported` | Property | | A `/Filter` array of millions of filters, each costing decoding whatever its data | ISO 32000 sets no limit, and each filter costs a few microseconds, LZW a table of 42 KB, whatever its data; a filter whose data decodes to itself — `/RunLengthDecode` over `FF FF`, an identity `/Crypt` — never runs out of it. `MaxFilterChainLength` (64) bounds the filters a stream's data passes through: decoding stops before the first past it, and the elements past it are not read, by decoding, `HasImageFilter` or the cross-reference stream's check alike. Data a filter decodes to nothing ends decoding. What a chain keeps of what it read is what the file did not write plainly, within the entries the guard lets run: nothing for a name the array holds, the name an element written as a reference leads to, and only the values of parameters that differ from none at all; an array of forty million names, which an object stream's member holds in 350 KB of file, ran a decode out of memory when the chain kept 32 bytes for each | `HostileInputTests.Decodes_a_stream_naming_a_million_filters_at_the_cost_of_as_many_as_the_guard_lets_run`, `Decodes_a_stream_naming_a_million_filters_without_keeping_anything_for_the_names_its_array_holds`, `Decodes_nothing_further_once_a_filter_has_decoded_a_stream_to_nothing`; `FilterDamageTests.Gives_a_dictionary_of_parameters_beside_a_chain_to_its_flate_and_its_lzw_steps_alike`, `Reads_a_parameter_past_an_int_as_the_int_nearest_it_rather_than_its_low_bits`; `FilterTests.Reads_nothing_past_the_most_filters_the_guard_lets_a_stream_pass_through`, `Says_a_stream_is_an_image_only_when_its_image_filter_stands_within_the_guard`, `Reads_the_filters_a_chain_names_through_references_in_their_order_crypt_filters_among_them`; `CrossReferenceRuleTests.A_cross_reference_stream_s_filters_past_the_guard_are_not_followed`; `ReaderLimitsTests.Reaching_a_guard_keeps_what_fits_and_names_the_property_that_lifts_it`, `A_stream_the_reader_reads_whose_filters_pass_the_guard_is_cut_before_its_data` | No | -| A guard's cut reported as damage by the filters after it | Every filter after one the bound stopped, or after the parser's window cut the data, is given data the reader cut: no lost tail or missing checksum, no rows longer than the data; a hexadecimal digit or an ASCII85 group the cut parted is dropped, not completed, and no end marker is missing from it. A fault, or a checksum that disagrees, in what was read stays the file's ([#199], [#141]) | `FilterDamageTests.Says_nothing_of_the_end_of_data_a_filter_before_it_stopped_at_the_bound`, `Reports_a_fault_in_the_data_a_filter_kept_though_a_filter_before_it_stopped_at_the_bound`, `Drops_the_unit_a_guard_of_the_reader_parted_rather_than_complete_it`, `Completes_the_unit_its_end_marker_closes_however_the_reader_cut_the_data_after_it`, `Undoes_a_predictor_over_the_rows_of_a_bounded_lzw_output_and_the_part_of_the_last_it_holds`; `FilterTests.Undoes_a_predictor_over_the_rows_of_a_bounded_output_and_the_part_of_the_last_it_holds`, `Leaves_the_partial_last_row_of_data_the_reader_did_not_cut_as_the_predictor_left_it` | Property | +| A guard's cut reported as damage by the filters after it | Every filter after one the bound stopped, or after `MaxStreamLength` cut the data, is given data the reader cut: no lost tail or missing checksum, no rows longer than the data; a hexadecimal digit or an ASCII85 group the cut parted is dropped, not completed, and no end marker is missing from it. A fault, or a checksum that disagrees, in what was read stays the file's ([#199], [#141]) | `FilterDamageTests.Says_nothing_of_the_end_of_data_a_filter_before_it_stopped_at_the_bound`, `Reports_a_fault_in_the_data_a_filter_kept_though_a_filter_before_it_stopped_at_the_bound`, `Drops_the_unit_a_guard_of_the_reader_parted_rather_than_complete_it`, `Completes_the_unit_its_end_marker_closes_however_the_reader_cut_the_data_after_it`, `Undoes_a_predictor_over_the_rows_of_a_bounded_lzw_output_and_the_part_of_the_last_it_holds`; `FilterTests.Undoes_a_predictor_over_the_rows_of_a_bounded_output_and_the_part_of_the_last_it_holds`, `Leaves_the_partial_last_row_of_data_the_reader_did_not_cut_as_the_predictor_left_it` | Property | +| A `/Filter`, a `/DecodeParms` or a step's parameter that leads to an object a guard cut or left out of the index, read as no filter, or as the `/Flate` a cut parted of `/FlateDecode`: an object stream's header read as stray bytes, its catalog looked for elsewhere, a sound file under a lowered limit judged broken | The chain of filters ends at such a value, through a chain of references too, as at the element past `MaxFilterChainLength` (`FilterChainEnd.Guard`): the steps before it run, the data is left encoded by the rest (`leftEncoded`), and nothing more is reported, the guard having been reported where it was reached. An object stream so left is hidden and its members are the guard's: a catalog among them is taken as read, and the validator reports `xref.checked-in-part`, not `xref.object-stream-broken`. Decoding says whether the stream's own data is cut — by `MaxStreamLength`, or by a filter stopped at `MaxDecodedStreamLength` —, which is what the readers of rows and members ask: a guard that loading a filter's value reaches is that value's, and hides none of the stream's own faults ([#263]) | `GuardCutValueTests.An_object_stream_whose_filter_values_a_guard_cut_is_hidden_and_its_members_taken_as_read`, `A_content_stream_whose_filter_a_guard_cut_comes_back_as_the_filters_before_it_decode_it`, `A_guard_a_filter_value_reaches_as_it_loads_is_not_the_stream_s_own_cut` | No | | An output buffer sized from the file | The first buffer is an estimate from the input, capped at the bound; it doubles and never grows past it | `FilterTests.A_filter_s_first_buffer_is_sized_by_its_bound_not_by_its_input`, `A_bounded_output_never_grows_past_its_bound_and_hands_back_a_full_buffer_as_it_is` | Yes | | Predictor parameters that overflow or size a row before a check | The row length is computed in 64 bits and compared with the data before anything is allocated | `FilterTests.Leaves_the_data_as_decoded_when_the_predictor_describes_rows_it_cannot_hold` | Seeds only | | An LZW code past the table | A fixed table of 4,096 codes. A code not yet defined stops decoding, keeps what came before and reports `filter.failed` | `FilterDamageTests.Decodes_an_lzw_code_up_to_the_next_one_to_define_and_stops_at_any_past_it`; `FilterTests.Stops_an_lzw_stream_at_a_code_it_has_not_defined_and_says_so` | Two seeds | | Data a filter cannot decode as its format says, read in silence | ASCII85, ASCIIHex and RunLength decode what breaks their format as other readers do and report it as `filter.failed`, each kind once a step, at the byte where the first lies: a byte that encodes nothing, passed over; an ASCII85 group past four bytes, a `~` no `>` follows, a last group of one digit, the `<~` PostScript writes. Data without its end marker — LZW's end-of-data code among them — decodes to its end: a repair when it ends after a whole unit, a warning when it ends inside one, an LZW code included. Counting costs nothing allocated; a filter stopped at the bound counts the faults of the part it read, and the end of data the reader cut or a filter stopped at the bound, a `~` whose `>` lies past the cut among it, is the guard's ([#141]) | `FilterDamageTests.Passes_over_the_bytes_that_encode_nothing_and_reports_where_the_first_lies`, `Reports_an_ascii85_group_whose_value_does_not_fit_in_four_bytes`, `Reads_an_ascii85_marker_out_of_place_as_other_readers_do_and_reports_it`, `Decodes_data_without_its_end_marker_to_its_end_and_says_whether_any_may_be_lost`, `Reports_no_missing_end_marker_where_a_filter_stopped_decoding_at_the_bound`, `Reports_no_missing_end_marker_where_a_filter_stopped_at_the_bound_completing_its_last_unit`, `Reports_no_missing_greater_than_sign_where_the_reader_cut_ascii85_data_after_its_tilde`, `Reports_no_end_marker_a_guard_of_the_reader_cut_off_and_a_fault_in_what_it_read`, `Decodes_an_lzw_stream_without_its_end_of_data_code_to_its_end_and_says_whether_a_code_was_cut`; `TextFilterRefereeTests` (qpdf) | No ([#176]) | | Damaged Flate data read again and again | A form is read again only after a fault, into the same output. The byte-at-a-time pass covers only the 8 KB piece the fault was met in (`FlateInput.ChunkLength`). A stream costs at most eight readings, three of them byte by byte, and a chain reads its input again only in the filter that met the fault: one that decodes nothing ends it. This is accepted: damaged streams read about 40 times slower, still linear in the file (#135, comment of 2026-09-30) | `FilterDamageTests.Reads_sound_data_once`, `Reads_again_a_byte_at_a_time_only_the_piece_of_input_the_fault_was_met_in`, `Reading_damaged_data_again_allocates_no_second_output_and_sound_data_nothing_to_read_it_again_with` | Yes | | The inflater's exceptions escaping | Only its complaints about the data (`InvalidDataException`, `IOException`) are faults; a preset-dictionary header is refused | `FilterDamageTests.Takes_only_the_inflater_s_complaints_about_its_data_as_faults`, `Reports_a_zlib_header_that_asks_for_a_preset_dictionary_instead_of_throwing` | Yes | -| Image codecs | DCT, JPX, JBIG2 and CCITT data is never decoded: the chain stops and the data comes back as it is. An unknown filter, or an element of `/Filter` that names none, is `filter.unsupported`, placed where the stream's data starts, and the chain stops there too: no filter after it reads data it cannot read ([#159], [#209]) | Structural; `FilterTests.Stops_a_chain_at_the_first_filter_it_does_not_know_and_reports_it_once`, `Ends_a_chain_at_an_element_that_names_no_filter_and_returns_what_the_filters_before_it_decoded` | — | +| Image codecs | DCT, JPX, JBIG2 and CCITT data is never decoded: the chain stops and the data comes back as it is, within `MaxStreamLength`. An unknown filter, or an element of `/Filter` that names none, is `filter.unsupported`, placed where the stream's data starts, and the chain stops there too: no filter after it reads data it cannot read ([#159], [#209]) | Structural; `FilterTests.Stops_a_chain_at_the_first_filter_it_does_not_know_and_reports_it_once`, `Ends_a_chain_at_an_element_that_names_no_filter_and_returns_what_the_filters_before_it_decoded` | — | Known gaps: @@ -452,8 +466,9 @@ Known gaps: ## What holds the defenses - **Hand-written hostile tests**, on every commit: `HostileInputTests`, `ReaderLimitsTests`, `ReaderTimingTests` - (a ratio, not a wall-clock budget), `WindowEdgeTests`, `StreamLengthTests`, `FilterTests`, `FilterDamageTests`, - `DiagnosticsTests`, and the validation rule tests. They hold the bounds a mutation of a small file cannot reach. + (a ratio, not a wall-clock budget), `WindowEdgeTests`, `StreamLengthTests`, `GuardCutValueTests`, `FilterTests`, + `FilterDamageTests`, `DiagnosticsTests`, and the validation rule tests. They hold the bounds a mutation of a small + file cannot reach. - **Budgets on real documents, and on a generated index.** `CorpusReadingTests` holds what indexing a 1,000-page document and walking its page tree allocate, and each operation on every corpus document — opening it, reading every object, walking its pages, validating a damaged one — to 20 s. `CorpusValidationTests.Validating_the_largest_document_stays_within_its_budget` @@ -711,6 +726,7 @@ The last column also names the questions they leave open, which that milestone s [#218]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/218 [#220]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/220 [#221]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/221 +[#224]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/224 [#229]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/229 [#230]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/230 [#237]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/237 @@ -720,6 +736,7 @@ The last column also names the questions they leave open, which that milestone s [#260]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/260 [#261]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/261 [#262]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/262 +[#263]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/263 [#273]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/273 [#274]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/274 [#275]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/275 diff --git a/docs/website/docs/concepts/reader-limits.md b/docs/website/docs/concepts/reader-limits.md index 4aaf1bfc..59c44925 100644 --- a/docs/website/docs/concepts/reader-limits.md +++ b/docs/website/docs/concepts/reader-limits.md @@ -37,7 +37,8 @@ it, and a file that kills the process is the worst outcome a document reader can ## Guards a valid file can reach Some of those bounds are reachable by a sound document: a large-format scan decodes past the 256 MB a stream may -decode to by default, a file saved incrementally a thousand times has a thousand cross-reference sections, and nothing +decode to by default, and stored without a filter its data runs past the 256 MB the reader reads of one stream at once; +a file saved incrementally a thousand times has a thousand cross-reference sections, and nothing in PDF 2.0 forbids more objects than the 8,388,607 the index holds by default. A bound like that is a **guard**. It is on by default, so an application that opens files it did not produce is protected without configuring anything; it is reported under its own `limit.*` code when a document reaches it; and its message names the option that lifts it, so that the document stays readable by whoever needs to read it. @@ -48,6 +49,13 @@ when it was the reader that stopped reading it. The bounds live on the options r two documents opened side by side can be read under different ones, and a stream decodes under the limits of the document it came from, however long after opening. +A value read through a reference to an object a guard cut, or left out of the index, is the guard's too. What the cut +left of it may be the start of what the file holds — `1` of `15`, `/Flate` of `/FlateDecode` —, so it is never judged +as a value the file gave: the data of a stream whose `/Length` is so named is found by its `endstream`, and its length +is not judged; decoding stops at a filter so named, or one whose parameters are, and leaves the rest of the data +encoded, as past the most filters the reader runs; and a cross-reference stream whose values lead to an object a guard +cut ends the chain as a section a guard cut does. + What a guard leaves out of the cross-reference index — the oldest sections of a long chain, the rows past a long table's bound, the objects past the most the index holds — is no damage either. Those objects read as null, unknown rather than missing, until you raise the limit; the reader does not rebuild the index for them, as it would for a file diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index 4322e8da..fa9c48e9 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -14,10 +14,11 @@ document instead, in [Read or refuse a document that reaches a limit](../guides/ | Property of `PdfReaderLimits` | Default | Reached by | What is kept | Code | |---|---|---|---|---| | `MaxDecodedStreamLength` | 256 MB | A decompression bomb, or a large-format scan: a 9,600 × 11,410 RGB map decodes to 313 MB. Of a chain of filters, the filters before the last share the bound for what they pass along together, and the last has its own; a `/Crypt` filter, which passes its data along as it is, is neither charged nor the last | The first bytes, up to the bound | `limit.decoded-stream` | +| `MaxStreamLength` | 256 MB | A stream read from a file whose encoded data is that long, read in one piece each time `GetRawBytes` or decoding asks for it: that map's image stored without a filter would be 313 MB. It bounds what is read, where `MaxDecodedStreamLength` bounds what filters produce: data decoding hands back as it is — no filter, an image filter, a filter the library does not know, one that could not decode it — is bounded by it alone | The data's first bytes, up to the bound; `Data.Length` and `RawLength` stay the length the file holds | `limit.stream-length` | | `MaxObjectLength` | 16 MB | An object that long, the data of a stream aside, written in the file or held by an object stream: an array of a million references | The object as far as the bound | `limit.object` | | `MaxXRefSectionLength` | 64 MB | A classic cross-reference table of more than about 3.3 million entries | The entries within the bound | `limit.xref-section-length` | | `MaxXRefSectionCount` | 1,024 | A document saved incrementally more than a thousand times | The newest sections | `limit.xref-section-count` | -| `MaxTrailerLength` | 64 KB | A trailer that long, or a cross-reference stream's dictionary: its `/Index` grows with every scattered update | The trailer as far as the bound | `limit.trailer` | +| `MaxTrailerLength` | 64 KB | A trailer that long, or a cross-reference stream's dictionary — its `/Index` grows with every scattered update —, never that stream's data | The trailer as far as the bound | `limit.trailer` | | `MaxIndexEntries` | 8,388,607 | A document of more objects than ISO 32000-1's Annex C advises; or a cross-reference stream of one-byte rows, of which 256 MB decode from a few hundred bytes | The object numbers indexed first: the newest sections', a rebuild's first | `limit.index-entries` | | `MaxFilterChainLength` | 64 | A `/Filter` array naming more filters, `/Crypt` among them: real streams name one to three, and a hostile array of millions costs decoding a few microseconds a filter, and some filters a table of tens of kilobytes, whatever their data | The data as the filters within the bound decoded it; the elements past it are not read | `limit.filter-chain` | @@ -32,7 +33,8 @@ bytes allocated an object, so a document near the bound takes about 24 s and 3 G validates uploads can lower it to the documents it accepts. A value of zero or less is refused with an `ArgumentOutOfRangeException`. A length above `Array.MaxLength` -is taken as `Array.MaxLength`: a decoded stream is held in one piece, and no runtime allocates a larger one. +is taken as `Array.MaxLength`: a decoded stream, and a stream's data read at once, are held in one piece, and no +runtime allocates a larger one. `PdfReaderOptions.Limits` refuses null with an `ArgumentNullException`. ## When a limit is reached @@ -48,7 +50,8 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. - An object, a cross-reference section or a trailer is reported once, however often it is read again — an object an object stream holds, once for each object and each definition of its object stream —; a full index, once a document. - What a limit leaves out of the index — the sections past `MaxXRefSectionCount`, the rows past - `MaxXRefSectionLength` or past what `MaxDecodedStreamLength` decodes of a cross-reference stream, the object + `MaxXRefSectionLength`, or past what `MaxStreamLength` reads or `MaxDecodedStreamLength` decodes of a + cross-reference stream, the object numbers past `MaxIndexEntries` — reads as null until you raise the limit. The reader does not rebuild the index for it, since the file may be sound, and the validator reports it as unknown, under `xref.checked-in-part`, rather than missing. When the trailer's `/Root` names an object so left out, the catalog reads as null too: the reader does not @@ -62,6 +65,18 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. object so cut lacks or holds, and takes a page tree node so cut to give what its pages inherit. An object header that runs past the bound, where a cross-reference entry places it, is where a read reaches the limit, and the validator says that entry was not checked. +- A stream whose data runs past `MaxStreamLength` is read up to the bound each time its data is asked for, by + `GetRawBytes` or by decoding, and its filters decode what was read as data the reader cut: that it runs out is not + reported as damage. Data that decoding hands back as it is comes back as far as the bound. The guard is reported once for + each stream, in the document's own diagnostics whatever you pass to `Decode`, at the offset its data starts at; with + `ThrowOnLimit` each read throws. In an object stream so cut, an object that lies past the cut is the guard's, unknown + until you raise the limit, as what any limit leaves out is: + + ```text + Warning limit.stream-length at 1904: A stream's data runs past 256 MB; only what lies within it was read. Raise + PdfReaderLimits.MaxStreamLength to read past it. + ``` + - A stream decoded past the bound is reported by the first of its filters that stopped there: each time it is decoded, in the diagnostics you pass to `Decode`; once, in the document's own — when you pass none, or the document's —, however often you, the reader or the validator decode it. The guard is counted each time all the same, and with `ThrowOnLimit` @@ -90,12 +105,28 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. cross-reference stream or an object stream the reader meets so is read as one a guard cut before its data: its rows, or its objects, are unknown until you raise the limit, as what any limit leaves out is. +- A value a stream is read with that leads to an object a guard cut, or left out of the index — its `/Length`, + `/Filter` or `/DecodeParms`, a value of a filter's parameters, directly or through a chain of references — is the + guard's, and nothing is reported of it but the guard. The data of a stream whose `/Length` is so named is found by + the search for its `endstream`, and its length is not judged: an integer the cut left, which may be the first digits + of a longer one, is only a place to look for `endstream` at. No `endstream` after the data of a stream whose + `/Length` names an object a guard cut is still the file's fault, and reported. Decoding stops at a filter so named, + or at the filter whose parameters are: the filters before it run, and the stream comes back as they decoded it, + still encoded by the rest, as past `MaxFilterChainLength`. An object stream the reader meets so is read as one a + guard cut before its data: its objects are unknown until you raise the limit, and when the catalog is among them the + reader does not look for another. A cross-reference stream whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` + leads to an object `MaxObjectLength` cuts while the chain is read is a section the guard cut: the chain ends there, + the index is not rebuilt, and the guard is reported once the chain is read, or thrown by `Open`. - A stream read from a document decodes under **that document's** limits, however long after opening you decode it. A stream you build in memory decodes under `PdfReaderLimits.Default`. - A classic trailer that ends within the window its cross-reference table was read through is read whole, whatever - its length. `MaxTrailerLength` bounds how far the reader follows one past that window. + its length. `MaxTrailerLength` bounds how far the reader follows one past that window, and a cross-reference stream's + dictionary, not its data: the data of one whose `/Length` the reader cannot read while it reads the chain is searched + for its `endstream` in the file, up to the next object the sections read so far place, or the end of the file. Data + no `endstream` ends before there makes its section malformed, a fault of the file and not of a guard. - Until the reader decodes streams a piece at a time, a stream that decodes past about 2 GB is cut there, and - reported, even under `Unbounded`. + reported, even under `Unbounded`. A stream's data is read in one piece, and data past about 2 GB is read up to + there, and reported under `limit.stream-length`, even under `Unbounded`. - The reader keeps the object streams it decoded within a budget of 32 MB — their decoded data, and their headers' tables —, and always keeps the one it decoded last, whatever its size. Raising `MaxDecodedStreamLength` raises how large that one can be. @@ -117,8 +148,9 @@ The exception comes from whichever operation reaches the limit: | Operation | When | |---|---| | `PdfDocument.Open` | While it indexes the file and reads its catalog | -| `GetObject`, or resolving a reference | When an object is too long, or the object stream that holds it decodes too far or names too many filters; when the read rebuilds the index, at any limit the rebuild reaches, once the rebuild is finished — the first reached: a limit the rebuild's scan reaches before the read's own, the read's own before one finishing the rebuild reaches; and each time it looks up an object a full index left out | -| `Decode` | When a stream decodes too far, or its decoding reaches the filter past `MaxFilterChainLength` | +| `GetObject`, or resolving a reference | When an object is too long, or the object stream that holds it runs past `MaxStreamLength`, decodes too far or names too many filters; when the read rebuilds the index, at any limit the rebuild reaches, once the rebuild is finished — the first reached: a limit the rebuild's scan reaches before the read's own, the read's own before one finishing the rebuild reaches; and each time it looks up an object a full index left out | +| `GetRawBytes` | When the stream's data runs past `MaxStreamLength`, each time it is read | +| `Decode` | When a stream's data runs past `MaxStreamLength`, the stream decodes too far, or its decoding reaches the filter past `MaxFilterChainLength` | | `PdfValidator.Validate` | As it reads, like any other caller; and before its first rule, at any limit the rebuild it may run then reaches, once that rebuild is finished | A document whose index had to be rebuilt finishes the rebuild before throwing, so it stays usable afterwards. @@ -132,6 +164,7 @@ the documents producers write. They are not options: |---|---|---| | Nesting of arrays and dictionaries | 128 levels | Real documents nest a handful of levels; the bound keeps a hostile file from exhausting the stack | | A reference that resolves to another reference | followed 32 times | Producers write the object itself; the bound breaks a chain that loops | +| Searches for a stream's `endstream` | four times the file's length, for a document's searches together | A valid stream's `endstream` follows its declared length: the file is read past the bytes an object is parsed through only for the data of a cross-reference stream whose `/Length` the chain cannot read, and for white space and comments after a declared length that run past a first read — about twice the file at most. Past the bound, a stream whose `endstream` the bytes read with its object do not show is not searched for further, and its report says so | | Objects loaded while loading another | 64 | An indirect `/Length` leads to an integer, and a stream resolves its filters and the parameters they read before decoding any of its data: no level holds decoded data while the next loads | | Object numbers | up to 2,147,483,647 | An object's number is an `int`, and ISO 32000-1's Annex C advises at most 8,388,607 indirect objects. A number past it is refused where it is read — a header, a reference, a subsection, a row, an object stream's header —, never narrowed to one that would name another object. An object is numbered from 1; 0 names the head of the free list, where a reference, a subsection or a row may give it | | An object stream's members | each read to the next one's start, the last to the end of the decoded data | ISO 32000-1 (7.5.7) has a header list its objects by increasing offset, so that each lies before the next one starts: a pair whose offset is not past the one before it, or whose object would start past the data, ends the header there, as a stray token does, and the objects listed before it are read. A member past `MaxObjectLength` is that limit's | From 379b8866bd8068281e216940cfc95991af4b1202 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 07:15:57 +0000 Subject: [PATCH 009/146] test: name PDFBox's zeroed object stream among the lengths qpdf takes otherwise The referee held that the reader and qpdf both took object 417's data of PDFBox's pdfbox3947 up to object 441's endstream, past a 32 KB block of zeros that erased its own: inside its window the reader did not bound its search. Its single search for a stream's end now stops at the next object the index places, in the bytes the reader holds as past them: the reader stops inside the block and keeps the declared 1,302 bytes, where qpdf, whose search no object bounds, reads 34,152. The expectation is corrected, not weakened: the stream joins the known disagreements, for qpdf's reason, as PDFium's urban planning report's object 695 did, and the remarks no longer say the reader's search is unbounded inside its window. The manifest's expectations of the document hold. Refs #174 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../StreamLengthRefereeTests.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/AdCodicem.Pdf.IntegrationTests/StreamLengthRefereeTests.cs b/tests/AdCodicem.Pdf.IntegrationTests/StreamLengthRefereeTests.cs index 8af1ec02..bb4bf40d 100644 --- a/tests/AdCodicem.Pdf.IntegrationTests/StreamLengthRefereeTests.cs +++ b/tests/AdCodicem.Pdf.IntegrationTests/StreamLengthRefereeTests.cs @@ -20,10 +20,8 @@ namespace AdCodicem.Pdf.IntegrationTests; /// qpdf reads the copy of an object its own index places, which is the reader's only when both indexes place the same /// one: a stream qpdf's index lacks — tiff2pdf's scans, whose chain qpdf does not follow —, or places elsewhere — the /// Atypon article's object 49, which the reader reads where its rebuilt index places it —, is another stream, and is not -/// compared. qpdf's search for endstream is not bounded by the next object, as the reader's past its window is; -/// where the two disagree for that reason, the stream is named below. Inside the window the reader does not bound its -/// search either, and the two agree even where both are wrong: PDFBox's zeroed object stream, object 417, takes object -/// 441's endstream, a zeroed stretch having erased its own. +/// compared. qpdf's search for endstream is not bounded by the next object, as the reader's is, in the bytes it +/// holds as past them (#174); where the two disagree for that reason, the stream is named below. /// /// [Collection(RefereeCollection.Name)] @@ -34,10 +32,15 @@ public class StreamLengthRefereeTests(RefereeContainer referee) /// planning report: a block of zeros erased the end of object 695's data, its endstream and the objects after /// it; qpdf takes the first endstream past the block, object 778's, and reads 3,292,376 bytes, where the reader /// stops at object 696, which the index as written places 21 bytes past the declared length, and keeps that length. + /// PDFBox's zeroed object stream: a 32 KB block of zeros erased object 417's endstream and the objects after it; + /// qpdf takes object 441's endstream, past the block, and reads 34,152 bytes, where the reader stops at the next + /// object the index places, inside the block, and keeps the declared 1,302 — it took 441's too until #174 bounded its + /// search within the bytes it holds as past them. /// private static readonly Dictionary KnownDisagreements = new(StringComparer.Ordinal) { ["remote/opf-format-corpus/jhove-hul-35-pdfium-urban-planning-report.pdf"] = [695], + ["remote/pdfbox/distiller6-zeroed-object-stream-pdfbox3947.pdf"] = [417], }; [Theory] From 759fc6a1fb792b4751954f0167c989a508e14f2f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 07:17:04 +0000 Subject: [PATCH 010/146] docs: guide a caller through MaxStreamLength and a stream's data read each time The reader-limits concepts, the guides to the reader's limits and its memory, and the diagnostics reference say what MaxStreamLength bounds, what limit.stream-length reports, and that a stream's raw data is read from the file each time it is asked for, kept by nothing. Refs #170 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/website/docs/concepts/reader-limits.md | 20 +++++++++++-- .../docs/guides/handle-reader-limits.md | 20 ++++++++++--- .../website/docs/guides/tune-reader-memory.md | 28 +++++++++++++++++-- docs/website/docs/reference/diagnostics.md | 2 +- 4 files changed, 60 insertions(+), 10 deletions(-) diff --git a/docs/website/docs/concepts/reader-limits.md b/docs/website/docs/concepts/reader-limits.md index 59c44925..bda180e3 100644 --- a/docs/website/docs/concepts/reader-limits.md +++ b/docs/website/docs/concepts/reader-limits.md @@ -37,8 +37,8 @@ it, and a file that kills the process is the worst outcome a document reader can ## Guards a valid file can reach Some of those bounds are reachable by a sound document: a large-format scan decodes past the 256 MB a stream may -decode to by default, and stored without a filter its data runs past the 256 MB the reader reads of one stream at once; -a file saved incrementally a thousand times has a thousand cross-reference sections, and nothing +decode to by default, and stored without a filter its data would run past the 256 MB the reader reads of one stream at +once; a file saved incrementally a thousand times has a thousand cross-reference sections; and nothing in PDF 2.0 forbids more objects than the 8,388,607 the index holds by default. A bound like that is a **guard**. It is on by default, so an application that opens files it did not produce is protected without configuring anything; it is reported under its own `limit.*` code when a document reaches it; and its message names the option that lifts it, so that the document stays readable by whoever needs to read it. @@ -88,6 +88,15 @@ reader keeps, 32 MB together, the one it decoded last, whatever its size, and th more than its output: a Flate filter allocates up to about five times what it decodes to, and holds two to three times that while it decodes. +Before its first filter runs, a stream's encoded data is read, and that read is bounded too. ISO 32000 sets no limit on +a stream's length, and data no filter decodes — none named, an image filter, one the library does not know, one that +could not decode it — comes back as it is, out of reach of `MaxDecodedStreamLength`, which bounds what filters produce. +`MaxStreamLength` bounds what is read of one stream's data at once, 256 MB by default; past it the data is cut, and its +filters decode what was read as data the reader cut. No stream keeps its data: it is read each time it is asked for, +lent where it lies when the document is held in memory, read into an array of its length from a file. A stream costs +its data only while it is read or decoded, and for as long as you keep what that gave you, and streams whose data share +a stretch of the file do not each hold a copy of it. + `PdfReaderLimits.Unbounded` takes every limit to the most the implementation can hold. It is for documents your own application produced, never for uploads: under it, a file can make the reader hold whatever it asks for, up to those maxima. @@ -98,7 +107,12 @@ Other bounds protect the reader itself — its stack, the chains of references i documents producers write: real documents nest a handful of levels, not 128, and producers write an object rather than a reference to a reference. An object stream's header lists its objects by increasing offset, so that a sound one's objects each lie before the next one starts: the reader reads each no further, and a header that breaks that -order ends where it breaks it. Those stay internal constants, since lifting them would let nothing more be read, and +order ends where it breaks it. A sound stream's `endstream` follows its declared length; where it does not, or the +`/Length` gives none, the reader searches the file for it past the bytes it parsed the object through, rather than parse +the object through a larger window, so that a stream's data never counts toward its object's length. A document's +searches read four times the file at most: a valid file is read past those bytes only for the data of a cross-reference +stream whose `/Length` the chain cannot read yet, and for white space and comments after a declared length, about twice +the file. Those stay internal constants, since lifting them would let nothing more be read, and each says where it is declared why no valid file reaches it. A new bound is classified when it is added ([ADR 34](/project/adr/every-valid-pdf-is-readable-and-the-readers-guards-are)): diff --git a/docs/website/docs/guides/handle-reader-limits.md b/docs/website/docs/guides/handle-reader-limits.md index 8f9d836d..878377fd 100644 --- a/docs/website/docs/guides/handle-reader-limits.md +++ b/docs/website/docs/guides/handle-reader-limits.md @@ -29,9 +29,9 @@ foreach (var entry in document.Diagnostics) } ``` -Reading is lazy: a stream reaches `MaxDecodedStreamLength` when it is decoded, so look after decoding, not only after -opening. And an object a limit left out of the cross-reference index — the oldest sections of a long chain, the objects -past `MaxIndexEntries` — reads as null, as an object the file lacks does: the `limit.*` entry is what tells them apart. +Reading is lazy: a stream reaches `MaxStreamLength` when its data is read, and `MaxDecodedStreamLength` when it is +decoded, so look after reading streams, not only after opening. And an object a limit left out of the cross-reference +index — the oldest sections of a long chain, the objects past `MaxIndexEntries` — reads as null, as an object the file lacks does: the `limit.*` entry is what tells them apart. ## Raise the limit the report names @@ -50,6 +50,17 @@ using var document = PdfDocument.Open("topographic-map.pdf", options); Every stream of that document decodes under its limits, however long after opening you decode it. +The two bounds on a stream are not interchangeable. A large image stored without a filter reaches `MaxStreamLength`, +the bound on what is read of one stream's data, and not `MaxDecodedStreamLength`, which bounds what filters produce. +Raise `MaxStreamLength` for it: + +```csharp +var options = new PdfReaderOptions +{ + Limits = PdfReaderLimits.Default with { MaxStreamLength = 512 * 1024 * 1024 }, +}; +``` + ## Read documents you trust without limits For documents your own application produced, lift every limit at once: @@ -88,7 +99,8 @@ catch (PdfLimitExceededException exception) Put the `try` around everything that uses the document, not only around `Open`: the exception comes from whichever operation reaches the limit — `Open` while it indexes the file, `GetObject` or resolving a reference when an object is -too long or lies past what a full index holds, `Decode` when a stream decodes too far, `Validate` as it reads. +too long or lies past what a full index holds, `GetRawBytes` or `Decode` when a stream's data runs past +`MaxStreamLength`, `Decode` when a stream decodes too far, `Validate` as it reads. ## See also diff --git a/docs/website/docs/guides/tune-reader-memory.md b/docs/website/docs/guides/tune-reader-memory.md index 41114b1b..80638c0c 100644 --- a/docs/website/docs/guides/tune-reader-memory.md +++ b/docs/website/docs/guides/tune-reader-memory.md @@ -1,6 +1,6 @@ --- title: Tune the reader's memory -description: Trade memory for fewer re-reads with the object cache, keep more diagnostics, and open documents without copying them. +description: Trade memory for fewer re-reads with the object cache, read a stream's data once, keep more diagnostics, and open documents without copying them. --- # Tune the reader's memory @@ -25,7 +25,31 @@ var options = new PdfReaderOptions using var document = PdfDocument.Open("catalog.pdf", options); ``` -Measure before and after: the cache only pays off when the same objects are read again. +Measure before and after: the cache only pays off when the same objects are read again. A stream in the cache holds +its dictionary and where its data lies, not the data: its size does not weigh on the cache. + +## Read a stream's data once + +A stream keeps none of its data: each call to `GetRawBytes` or `Decode` reads it again. Opened from a path, each call +reads the data from the file into an array of its length; opened from a stream or from bytes, which the document holds +in memory, `GetRawBytes` lends the bytes where they lie, without a copy. When you use the same data more than once, +keep what one call gave you rather than call again: + +```csharp +using AdCodicem.Pdf.Objects; + +if (document.GetObject(new PdfObjectId(12)) is PdfStream stream) +{ + var data = stream.Decode(); // read and decoded once + + Index(data); + Render(data); +} +``` + +What one call reads is bounded by `PdfReaderLimits.MaxStreamLength`, 256 MB by default: past it the data is cut, and +`limit.stream-length` reported. [Read or refuse a document that reaches a limit](handle-reader-limits.md) shows how to +raise it. ## Keep more diagnostics diff --git a/docs/website/docs/reference/diagnostics.md b/docs/website/docs/reference/diagnostics.md index 69c87fde..3efaa92c 100644 --- a/docs/website/docs/reference/diagnostics.md +++ b/docs/website/docs/reference/diagnostics.md @@ -102,7 +102,7 @@ Codes are stable: they are part of the public contract, because callers filter o | `xref.rebuilt` | The index was rebuilt by scanning the whole file | | `xref.offset-adjusted` | An object, or a cross-reference section the chain names, was not where the file said, and was found nearby | | `xref.section-missing` | A cross-reference section `/Prev` or `/XRefStm` names holds nothing that reads as a section, nor does any place nearby, lies outside the file, or is named by a value that is not an offset; what only it indexed is found by rebuilding the index when it is asked for. Reported at the offset the section was named at, or, when that offset lies outside the file or the value is not an offset, at the section whose trailer named it | -| `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, `/Size` or data cannot give rows, or whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` is a reference no section read before it places where it can be read without loading it — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message | +| `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, `/Size` or data cannot give rows, or whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` is a reference no section read before it places where it can be read without loading it, or whose `/Length` is such a reference and no `endstream` follows its data before the next object the sections read so far place, or before the end of the file — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message. A value that names an object one of the reader's limits cut is the guard's, not the file's: the section is not reported, nor looked for nearby, the chain ends there, nothing is rebuilt, what only that section and the older ones index reads as null, and the guard is reported once the chain is read | | `xref.chain-cycle` | The chain of previous sections looped: a `/Prev` or `/XRefStm` names an offset the chain has already named, whether it read a section there or not. Reported at that offset, or, when it lies outside the file, at the section whose trailer named it | | `xref.entry-out-of-range` | An object's entry places it outside the file — past its end, before its start, or past what a `long` holds once the header's offset is added —; reported with no position, the offset in the message, exactly. A section named outside the file is `xref.section-missing`'s, or the rebuild's when `startxref` names it | | `stream.length-invalid` | A stream's `/Length` is not where its data ends: its `endstream` lies elsewhere and ends the data, or the `/Length` is no length — absent, not a non-negative integer, or naming an object the file lacks or that could not be read — and the `endstream` ends the data, or no `endstream` follows the declared length — none before the next object or the end of the file, or none looked for once the document's searches read as much as they may —, and that length is kept. A cross-reference stream whose `/Length` the chain could not read, its data taken up to its first `endstream`, has it read once the chain is: a `/Length` that is no length, or that no `endstream` confirms, is reported as any stream's, and one an `endstream` past the one the chain stopped at confirms is reported with the rows the chain left unread | From 9051f5d963e362dccfdaf37b7d6faf89b2177990 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 07:34:16 +0000 Subject: [PATCH 011/146] docs: describe where a stream ends and what a guard's cut leaves in the references The diagnostics, object-model and validation-rules references, the concepts of diagnostics, lazy reading and validation, the threat model and M02 describe the stream-end rules, the values a guard cut or left out of the index, and the header rule. The XML documentation of the codes, the pipeline, the reader, PdfStream, Decode and the endobj rule says the same. Refs #174, #224, #230, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/milestones/M02.md | 28 ++++- docs/threat-model.md | 6 +- docs/website/docs/concepts/diagnostics.md | 103 +++++++++++------- docs/website/docs/concepts/lazy-reading.md | 62 ++++++++--- docs/website/docs/concepts/validation.md | 14 ++- docs/website/docs/reference/diagnostics.md | 63 +++++++---- docs/website/docs/reference/object-model.md | 25 ++++- .../docs/reference/validation-rules.md | 24 ++-- .../Diagnostics/PdfDiagnosticCodes.cs | 12 +- .../IO/Filters/PdfFilterPipeline.cs | 6 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 13 +-- src/AdCodicem.Pdf/Objects/PdfStream.cs | 11 ++ .../Objects/PdfStreamDecodingExtensions.cs | 3 +- .../Validation/Rules/EndObjMissingRule.cs | 6 +- 14 files changed, 261 insertions(+), 115 deletions(-) diff --git a/docs/milestones/M02.md b/docs/milestones/M02.md index 3300dfcb..66491628 100644 --- a/docs/milestones/M02.md +++ b/docs/milestones/M02.md @@ -100,6 +100,24 @@ and none lost is no finding ([#275]); and an object an object stream writes as a stream, and the definition of it, that the chain's index places: an object a rebuild serves from another, which does not list it or writes it as a stream, reads as null with no finding yet ([#276]). +Where a stream's data ends is settled once for each stream, by the same rules wherever the parser's window ends: a +declared length that an `endstream` follows, past white space and comments however many, is the stream's; any other, or +none, goes to one search for the first `endstream`, up to the nearer of the first object header and the next object the +index as written places, the object's `endobj` standing in for it when none lies before; and a stream's data never +counts toward `MaxObjectLength` ([#174]). A cross-reference stream whose `/Length` the chain cannot read is searched +alike past the window, with no header stopping it; where the section's own row for that `/Length` lies among rows that +spell `endstream`, the data is taken again to the length it gives, and data no `endstream` ends before the next object +the sections read so far place makes the section malformed, the file's fault and no guard's ([#220]). The next object's +header ends every array and dictionary an object, or a trailer, left open, and the object lacks its `endobj` ([#230]). +A stream's `/Length`, `/Filter` or `/DecodeParms`, or a filter's parameter, that leads to an object a guard cut or left +out of the index is the guard's, never judged as a value the file gave: the length is not judged, and the data is found +by its `endstream`; the filters end there, the rest of them left undone, and an object stream so left is hidden as one a +guard cut, `xref.checked-in-part`'s rather than `xref.object-stream-broken`'s. A cross-reference stream whose `/W`, +`/Index`, `/Size`, `/Filter` or `/DecodeParms` leads to an object `MaxObjectLength` cuts while the chain is read ends the +chain as a section a guard cut, and nothing is rebuilt ([#224], [#263]). A stream's data is read each time it is asked +for and kept by nothing, at most `MaxStreamLength` of it at once, a guard: validating object streams whose data share a +stretch of the file holds no copy of it for each ([#170]). + ## Slices 1. Findings, report, rule engine, profile plumbing, and a single trivial rule end to end. **Done on @@ -227,8 +245,8 @@ dictionary listed as a kid is read as a page without `/Parent`, `/MediaBox` or ` `/Contents` naming an object the file lacks (`t02-01-004`, `t02-02-001`, `t02-03-011`) is `object.reference-missing`, a warning, and `t02-01-004`'s page, which no tree lists, `page-tree.page-orphaned`. Two content-stream cases whose `stream` keyword is missing, or that have bytes between `endstream` and `endobj` (`t02-05-01-013`, `-017`), earn -`object.endobj-missing` for what follows their value; one without `endstream` (`-014`) leaves where its object ends -unknown, and its stream is the stream rules' to judge. +`object.endobj-missing` for what follows their value; one without `endstream` (`-014`) keeps the 68 bytes its `/Length` +declares, which its object's `endobj` follows ([#174]), and its stream is the stream rules' to judge. The object-shape rules answered for the rest of the catalog, page tree and page object cases. A root typed `/Pagez` (`t02-02-009`), a page typed `/Font` (`t02-03-002`) and a catalog of the wrong `/Type` (`t02-01-006`) earn @@ -406,12 +424,18 @@ By Diátaxis section, since [ADR 47](../adr/0047-the-user-documentation-follows- [#154]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/154 [#160]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/160 [#161]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/161 +[#170]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/170 +[#174]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/174 [#175]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/175 [#181]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/181 [#183]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/183 [#200]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/200 [#202]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/202 +[#220]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/220 +[#224]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/224 +[#230]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/230 [#237]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/237 +[#263]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/263 [#274]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/274 [#275]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/275 [#276]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/276 diff --git a/docs/threat-model.md b/docs/threat-model.md index a7184c37..b6aacfda 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -261,7 +261,7 @@ Known gaps: - [#163] Nesting past 128 levels, and loads nested past 64, are cut though a valid file can reach both. They become guards. - [#262] A reference a guard cuts between its generation and its `R` reads as two integers. -- Its issue is still to be filed: a name, an integer or a real that a guard's cut parts is kept as the part read — +- [#279] A name, an integer or a real that a guard's cut parts is kept as the part read — `/Flate` of `/FlateDecode`, `1` of `15`, `1.0` of `1.5` — by every reader of the object it cut. A stream's `/Length` and the values its filters read take such an object for the guard's ([#224], [#263]); a caller reading it is told of the cut, not of the token it parted. @@ -360,7 +360,7 @@ Known gaps: Known gaps: -- Its issue is still to be filed: streams whose data share a stretch of the file each read that stretch again from a +- [#280] Streams whose data share a stretch of the file each read that stretch again from a source not held in memory, whenever their data is asked for. What overlapping streams cost is time rather than memory, bounded for each read by `MaxStreamLength` alone; a check for overlapping data is proposed. - [#138] A stream searched for its `endstream` can keep a length that depends on the object read first. @@ -742,3 +742,5 @@ The last column also names the questions they leave open, which that milestone s [#275]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/275 [#276]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/276 [#277]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/277 +[#279]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/279 +[#280]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/280 diff --git a/docs/website/docs/concepts/diagnostics.md b/docs/website/docs/concepts/diagnostics.md index 0e9dd1e6..b982649c 100644 --- a/docs/website/docs/concepts/diagnostics.md +++ b/docs/website/docs/concepts/diagnostics.md @@ -26,46 +26,65 @@ not how wrong the file is. The reader parses an object through a window of the file — 8 KB to start with — and reads it again through a larger one when it runs past the edge. What the smaller window saw there, such as a string -without its end or a stream without its `endstream`, is dropped with that attempt, so the report says what -the object holds, not where a window happened to end. When `MaxObjectLength` allows no larger window, the -object is kept as far as it was read and the guard is reported: what the parse met before the edge — a stray -token, a key that is not a name — is reported beside it, and nothing of the token the edge cut, nor of what that token -leaves open, is. - -A stream's declared length is checked against the `endstream` that must follow it. When the data runs past -the window, the reader asks the file for the few bytes after the declared length rather than reading the -data, and they tell it whether `endobj` follows too. When no `endstream` is there, the first one after the -start of the data ends it, and where it is looked for depends on where the declared length ends: - -- **Inside the window, or nowhere** — a `/Length` that gives no length —, the first `endstream` after the - data is taken wherever it lies, past the header of the next object too: the reader looks in the window, - and in a larger one while none is found, as far as `MaxObjectLength` allows. -- **Past the window**, the reader searches the file from the start of the data up to the next object — - the one the file's index places as the file wrote it (as the reader rebuilt it when the document opened, if - the file wrote none), or the first object header, `N G obj`, the file's bytes hold, whichever comes first —, - or up to the end of the file. - Stopping at the next object keeps a later object's `endstream` from ending this stream, which is what a - stretch of zeros that erased the end of one and the objects after it would otherwise do, or an object whose - entry the index lost; when none lies before it, the declared length is kept, and the report says so. Text - in a damaged stream's data that reads as an object header stops the search as well, and the declared - length is kept: an object number and a generation the parser takes, however many zeros lead them, white - space between the three tokens, however much, and white space, a delimiter, or the start or the end of - the search around them. A comment between the tokens makes no header here, as it makes none for the scan - that rebuilds a damaged index; nor does a regular character glued before the number, though that scan - takes the digits after it for one. - -The search is not one of the reader's limits: a valid file's `endstream` follows its length, so only a -damaged file is searched, and once for each stream. Where it stops depends on the file's bytes and on its -index as the file wrote it, which the reader rebuilding or correcting its own index later does not change: -in the 401 documents of the project's test corpus measured, each stream takes the same length whatever was -read before it — though a damaged index can still serve another copy of an object, at another offset, -according to what was read first. A file crafted for it can still make what a stream holds depend on which -object was asked for first. What the searches of one document read together is bounded: none starts once -they have read four times the file's length. They read twice the file at most — 0.35 of it at most in the -test corpus — unless their stretches overlap: one object's header inside another's dictionary, or a header -the search does not take — a regular character glued before its number — that the index as the file wrote -it does not place. Past the bound, a stream keeps its declared length without a search, and the report says -so. +without its end, is dropped with that attempt, so the report says what the object holds, not where a window +happened to end. When `MaxObjectLength` allows no larger window, the object is kept as far as it was read and the +guard is reported: what the parse met before the edge — a stray token, a key that is not a name — is reported beside +it, and nothing of the token the edge cut, nor of what that token leaves open, is. A stream's data is no part of what +a window must hold: where it ends is found in the file, whatever window its dictionary was read through, and it never +counts toward `MaxObjectLength`. + +A stream's declared length is confirmed by the `endstream` that must follow it, after any white space and comments, +however many: past the window, the reader asks the file for the bytes after the declared length rather than reading +the data, reading on as far as white space and comments run, and they tell it whether `endobj` follows too. Any other +stream — its length unconfirmed, or a `/Length` that gives none — is searched once for the first `endstream` after the +start of its data, which ends it: in the bytes the parser holds, then in the file, up to the next object — the one the +file's index places as the file wrote it (as the reader rebuilt it when the document opened, if the file wrote none), +or the first object header, `N G obj`, the file's bytes hold, whichever comes first —, or up to the end of the file. + +Stopping at the next object keeps a later object's `endstream` from ending this stream, which is what a stretch of +zeros that erased the end of one and the objects after it would otherwise do, or an object whose entry the index lost. +Text in a damaged stream's data that reads as an object header stops the search as well: an object number and a +generation the parser takes, however many zeros lead them, white space between the three tokens, however much, and +white space, a delimiter, or the start or the end of the search around them. A comment between the tokens makes no +header here, as it makes none for the scan that rebuilds a damaged index; nor does a regular character glued before +the number, though that scan takes the digits after it for one. Neither `xref` nor `trailer` stops a search: the +stream of a classic file's last object, which no object follows, is searched through the table and the trailer to the +end of the file. + +With no `endstream` before the stop, the object's `endobj` plays its part. A declared length that ends before the stop, +and that the object's `endobj` follows, is kept: the stream lost only its `endstream`. Otherwise, the object's own +`endobj`, when only white space separates it from the stop, ends the data, the end-of-line before it left out. Both are +reported as `stream.truncated`. Without one, a declared length that ends before the stop is kept, and whether the +object ends with its `endobj` is not judged; and the data otherwise runs to the stop, a declared length past it cut +there, the object judged to lack its `endobj` unless what is taken holds one. Both are reported as +`stream.length-invalid`, but data that runs to the end of the file as `stream.truncated`. + +A `/Length` that names an object one of the reader's limits cut is no length the file gave: what the cut left of it — +null, or `1` of a `15` it parted — is the guard's. An integer read before the cut is only a place to look for +`endstream` at, and the data is otherwise found by its `endstream`, as for a stream with no length; nothing is said of +the length, while a missing `endstream` is still the file's, and reported. One that names an object a guard left out of +the index is not judged either. + +A cross-reference stream whose `/Length` the chain cannot read yet, which a valid file may hold, is searched as well, up +to the next object the sections read so far place, or the end of the file, and its length read once the chain is. Its +rows may spell an object header and end in a carriage return: no header stops its search, and a carriage return before +the line feed that precedes `endstream` is kept. Data with no `endstream` before that stop ends nowhere the section +says, and its rows are not read. + +The search is not one of the reader's limits: a valid file's `endstream` follows its length, so only a damaged stream is +searched — and a cross-reference stream whose `/Length` the chain cannot read yet —, each once. Where it stops depends +on the file's bytes and on its index as the file wrote it, which the reader rebuilding or correcting its own index later +does not change: in the 401 documents of the project's test corpus measured, each stream takes the same length whatever +was read before it — though a damaged index can still serve another copy of an object, at another offset, according to +what was read first. A file crafted for it can still make what a stream holds depend on which object was asked for +first. What the searches of one document read together is bounded: none starts once they have read four times the file's +length. For a valid file, what they read — its cross-reference streams searched, the white space after a declared length +read on — comes to about twice the file at most, and the searches of damaged streams read twice the file at most — 0.35 +of it at most, all searches together, in the test corpus — unless their stretches overlap: one object's header inside +another's dictionary, or a header the search does not take — a regular character glued before its number — that the +index as the file wrote it does not place. Past the bound, only the bytes the parser holds are searched: a declared +length that ends before the next object is kept, the data otherwise taken up to it or to the end of the file, and the +report says so. ## Damaged stream data @@ -113,8 +132,8 @@ report, and reports nothing. A document's stream never decodes in silence. ## The reader's limits are not faults -Growing windows, and decoding, are bounded. The `limit.*` codes report the reader's own limits, not -faults of the file: a valid document can reach them — a large-format scan decodes past the 256 MB a stream +Growing windows, reading a stream's data, and decoding are bounded. The `limit.*` codes report the reader's own limits, +not faults of the file: a valid document can reach them — a large-format scan decodes past the 256 MB a stream may decode to by default. What fits within the limit is kept, and the limit replaces what the parser met where it stopped. [Reader limits](reader-limits.md) explains why they exist and why they are on by default. diff --git a/docs/website/docs/concepts/lazy-reading.md b/docs/website/docs/concepts/lazy-reading.md index f0e99192..3f8b0c80 100644 --- a/docs/website/docs/concepts/lazy-reading.md +++ b/docs/website/docs/concepts/lazy-reading.md @@ -17,7 +17,11 @@ Opening a document builds an index, finds the catalog, and stops there. stream. Nothing is loaded meanwhile: a value a section refers to — a cross-reference stream's `/W` or `/Length` written as a reference, a `/Prev` — is read only where a section already read places it, so that the map the chain gives is the file's own. A `/Length` no section read before places is read once the chain - is, and checked against the data the chain took up to its `endstream`. + is, and checked against the data the chain took up to the first `endstream` after the data's start, searched + for in the file up to the next object the sections read so far place, or its end — unless the section's own row + for that `/Length` lies among the rows the data holds: those rows spelled `endstream`, the object is read where + the row places it, without being kept, and the data is taken again to the length it gives when an `endstream` + confirms it. 4. Resolves the trailer's `/Root`, which reads the document catalog, and decodes the object stream that holds it when one does. When `/Root` leads to no catalog, it loads the objects the map holds, one after the other, until one is a catalog (`trailer.root-recovered`), and rebuilds the map when none is. @@ -51,16 +55,18 @@ can take 436 MB. An object the map does not hold is one the file does not define, and a reference to it reads as null, as the specification says — or one a limit left out of the map, which reads as null too, until the limit is raised: the -sections past `MaxXRefSectionCount`, the rows past `MaxXRefSectionLength` or past what `MaxDecodedStreamLength` -decodes of a cross-reference stream, the numbers past `MaxIndexEntries`. That is no damage, and the map is not rebuilt -for it. The map is rebuilt by scanning the whole file, once at most, and never while the chain is read: +sections past `MaxXRefSectionCount`, the rows past `MaxXRefSectionLength` or past what `MaxStreamLength` reads or +`MaxDecodedStreamLength` decodes of a cross-reference stream, a cross-reference stream whose `/W`, `/Index`, `/Size`, +`/Filter` or `/DecodeParms` leads to an object `MaxObjectLength` cuts and the sections older than it, the numbers past +`MaxIndexEntries`. That is no damage, and the map is not rebuilt for it. The map is rebuilt by scanning the whole file, +once at most, and never while the chain is read: - at opening, when `startxref` cannot be found or read, when the section it names cannot be read, when the chain indexes nothing, or when `/Root` leads to no catalog and no object of the map is one; - when an object the map lacks is asked for, by opening or after it, while the map may have lost entries: a cross-reference section the chain names cannot be found, or a `/Prev` or `/XRefStm` is not an offset — one written as a reference no section read before it places included — (`xref.section-missing`); one is found and - cannot be read (`xref.section-unreadable`), a value its rows need written as such a reference among the causes; the chain loops back on itself (`xref.chain-cycle`); a row of a classic table cannot be read, which ends its subsection; a row gives an object + cannot be read (`xref.section-unreadable`), a value its rows need written as such a reference, or a `/Length` so written whose data no `endstream` ends before the next object or the end of the file, among the causes; the chain loops back on itself (`xref.chain-cycle`); a row of a classic table cannot be read, which ends its subsection; a row gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, a field wider than 8 bytes whose leading bytes are not all zero —, which refuses that row alone, no older section's row standing for @@ -101,16 +107,38 @@ chain read for the numbers the scan did not find, rather than losing them. ## What it deliberately does not do Past the index and the catalog, nothing is parsed until something asks for it, and — short of a rebuild, which -scans the whole file — stream data is not even read from disk. A `PdfStream` holds the offset and length of its -bytes; the bytes arrive when you call `Decode()`, and not before. A test holds this on a document of about -500 KB, nearly all of it one page's content: opening it reads less than 200,000 bytes of the file, and reading -that content's bytes afterwards reads at least its 500,000 more. - -When it is decoded, a stream decodes under the [limits](reader-limits.md) of the document it came from, -however long after opening — so with `ThrowOnLimit` set, `Decode()` can throw long after `Open` returned. -What decoding meets — a stream that lost its tail, a limit reached — is reported when it is met, in the -diagnostics you pass to `Decode`, or in the document's own when you pass none: a damaged stream nobody -decodes is never reported, since nothing reads it. +scans the whole file, and of the search for where a damaged stream's data ends, below — stream data is not even read +from disk. A `PdfStream` holds the offset and length of its bytes; the bytes arrive when you call `GetRawBytes()` or +`Decode()`, and not before. A test holds this on a document of about 500 KB, nearly all of it one page's content: +opening it reads less than 200,000 bytes of the file, and reading that content's bytes afterwards reads at least its +500,000 more. + +Nor are the bytes kept once they have arrived: neither the stream nor the object cache holds them, and each call reads +them again. A document held in memory — opened from bytes or from a stream — lends them where they lie, without a copy; +one opened from a path reads them from the file into an array of their length. What a call hands back is the caller's +to keep or let go, so that memory follows the streams being read or decoded, not those read before, and streams whose +data share a stretch of the file — objects that overlap — do not each hold a copy of it. What one call reads is bounded +by `PdfReaderLimits.MaxStreamLength`, 256 MB by default: past it the data is cut, its filters decode what was read as +data the reader cut, and `limit.stream-length` is reported; `RawLength` stays the length the file holds. + +Finding where a stream's data ends reads little of it. The reader parses an object through a window of the file, and +confirms the declared `/Length` by the `endstream` that must follow it, after white space and comments, however many: +past the window, it asks the file for the bytes after the declared length rather than reading the data, and reads on +as far as white space and comments run. A length nothing confirms, or a `/Length` that gives none, sends the reader on +one search for the first `endstream` after the data's start: in the bytes the parser holds, then in the file, up to the +nearer of the first object header the bytes hold and the next object the index as the file wrote it places, or the end +of the file. With no `endstream` before that stop, the object's `endobj` plays its part. The search is made once for +each stream, however often the stream is parsed again; no window grows to find an `endstream`, so that a stream's data +never counts toward `MaxObjectLength`; and the searches of a document read four times the file at most — a valid +file's, about twice, for the white space after a declared length and the cross-reference streams whose `/Length` the +chain cannot read yet. What each case keeps, and reports, is in [Diagnostics](diagnostics.md#where-a-stream-ends). + +When it is read or decoded, a stream works under the [limits](reader-limits.md) of the document it came from, +however long after opening — so with `ThrowOnLimit` set, `GetRawBytes()` and `Decode()` can throw long after `Open` +returned. What decoding meets — a stream that lost its tail, a limit its filters reached — is reported when it is met, +in the diagnostics you pass to `Decode`, or in the document's own when you pass none; data cut at `MaxStreamLength` is +reported in the document's own, once for each stream. A damaged stream nobody decodes is never reported, since nothing +reads it. Holding a stream as an offset and a length is what will let a copy between documents — merging, assembling, stamping — move its **encoded** bytes as they are, with no decompress/recompress cycle in between, since nothing @@ -124,6 +152,10 @@ Beyond that bound, an object read again is read from the file again. That is the stays predictable, and the pathological case is slower rather than fatal. The bound is an option, and [Tune the reader's memory](../guides/tune-reader-memory.md) says when to move it. +A stream's data makes the same trade without a cache: no stream keeps it, cached or not, so that data asked for twice +is read twice, and, through `Decode()`, decoded twice. A caller that needs it more than once keeps what one call gave, +as [the same guide](../guides/tune-reader-memory.md#read-a-streams-data-once) shows. + The same options carry the [reader limits](reader-limits.md), which bound what one file may make the reader hold. diff --git a/docs/website/docs/concepts/validation.md b/docs/website/docs/concepts/validation.md index c99fef21..e67637cb 100644 --- a/docs/website/docs/concepts/validation.md +++ b/docs/website/docs/concepts/validation.md @@ -154,8 +154,12 @@ before the first rule —, one rule also looks at every object it holds, to name The document's [reader limits](reader-limits.md) apply to validation too. An object the reader cut at a limit is not a fault of the file, so a rule meeting one judges nothing of what it lacks or holds, or says, as information, that it could not check it whole, rather than calling the file broken; opening the document with a raised -limit lets it check the rest. The page tree rules take a node so cut to give what its pages inherit, and leave its -`/Count`, the value of those above it and the pages the tree may leave out unjudged; the reader takes a catalog so cut for the -catalog, as it read it, rather than look for another, and does not look for one either when a limit left the object -`/Root` names out of the index. Validation -throws only for the caller's own mistakes, and for a limit when the document was opened to throw on one. +limit lets it check the rest. The same holds of a value that leads to such an object: a cross-reference stream whose +`/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` leads to one is said not checked rather than malformed, and an +object stream whose `/Filter` or `/DecodeParms` leads to one, or to an object a limit left out of the index, is left +encoded, the objects it holds said not checked rather than lost +([#263](https://github.com/AdCodicem/AdCodicem.Pdf/issues/263)). The page tree rules take a node so cut to give what its +pages inherit, and leave its `/Count`, the value of those above it and the pages the tree may leave out unjudged; the +reader takes a catalog so cut for the catalog, as it read it — one such an object stream holds included —, rather than +look for another, and does not look for one either when a limit left the object `/Root` names out of the index. +Validation throws only for the caller's own mistakes, and for a limit when the document was opened to throw on one. diff --git a/docs/website/docs/reference/diagnostics.md b/docs/website/docs/reference/diagnostics.md index 3efaa92c..5833984c 100644 --- a/docs/website/docs/reference/diagnostics.md +++ b/docs/website/docs/reference/diagnostics.md @@ -78,7 +78,7 @@ The list `PdfDocument.Diagnostics` holds, in the order the entries were recorded | none, a stream read from a document | The document's own `Diagnostics` | | none, a stream built in memory | Nowhere: nothing is reported | -A stream that is never decoded is never reported: nothing reads it. +A stream that is never decoded reports no fault of its decoding: nothing decodes it. ## Severities @@ -102,17 +102,17 @@ Codes are stable: they are part of the public contract, because callers filter o | `xref.rebuilt` | The index was rebuilt by scanning the whole file | | `xref.offset-adjusted` | An object, or a cross-reference section the chain names, was not where the file said, and was found nearby | | `xref.section-missing` | A cross-reference section `/Prev` or `/XRefStm` names holds nothing that reads as a section, nor does any place nearby, lies outside the file, or is named by a value that is not an offset; what only it indexed is found by rebuilding the index when it is asked for. Reported at the offset the section was named at, or, when that offset lies outside the file or the value is not an offset, at the section whose trailer named it | -| `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, `/Size` or data cannot give rows, or whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` is a reference no section read before it places where it can be read without loading it, or whose `/Length` is such a reference and no `endstream` follows its data before the next object the sections read so far place, or before the end of the file — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message. A value that names an object one of the reader's limits cut is the guard's, not the file's: the section is not reported, nor looked for nearby, the chain ends there, nothing is rebuilt, what only that section and the older ones index reads as null, and the guard is reported once the chain is read | +| `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, `/Size` or data cannot give rows, or whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` is a reference no section read before it places where it can be read without loading it, or whose `/Length` is such a reference and no `endstream` follows its data before the next object the sections read so far place, or before the end of the file, or none was looked for once the document's searches for `endstream` read as much of the file as they may — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message. A value that names an object one of the reader's limits cut is the guard's, not the file's: the section is not reported, nor looked for nearby, the chain ends there, nothing is rebuilt, what only that section and the older ones index reads as null, and the guard is reported once the chain is read | | `xref.chain-cycle` | The chain of previous sections looped: a `/Prev` or `/XRefStm` names an offset the chain has already named, whether it read a section there or not. Reported at that offset, or, when it lies outside the file, at the section whose trailer named it | | `xref.entry-out-of-range` | An object's entry places it outside the file — past its end, before its start, or past what a `long` holds once the header's offset is added —; reported with no position, the offset in the message, exactly. A section named outside the file is `xref.section-missing`'s, or the rebuild's when `startxref` names it | -| `stream.length-invalid` | A stream's `/Length` is not where its data ends: its `endstream` lies elsewhere and ends the data, or the `/Length` is no length — absent, not a non-negative integer, or naming an object the file lacks or that could not be read — and the `endstream` ends the data, or no `endstream` follows the declared length — none before the next object or the end of the file, or none looked for once the document's searches read as much as they may —, and that length is kept. A cross-reference stream whose `/Length` the chain could not read, its data taken up to its first `endstream`, has it read once the chain is: a `/Length` that is no length, or that no `endstream` confirms, is reported as any stream's, and one an `endstream` past the one the chain stopped at confirms is reported with the rows the chain left unread | -| `stream.truncated` | A stream has no `endstream` before the end of the file, or before the `endobj` that follows its data, and its data runs to the end of the file | +| `stream.length-invalid` | A stream's `/Length` is not where its data ends: its `endstream` lies elsewhere and ends the data; or the `/Length` is no length — absent, not a non-negative integer, or naming an object the file lacks or that could not be read — and the `endstream` ends the data; or no `endstream` lies before the next object or the end of the file, no `endobj` of its object ends the data, and the data is the declared length, when it ends before that stop, or runs up to the next object; or the document's searches for `endstream` have read as much of the file as they may, and the data is taken without a search. A `/Length` that names an object a guard cut, or left out of the index, is not judged. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case. A cross-reference stream whose `/Length` the chain could not read, its data taken up to its first `endstream`, has it read once the chain is: a `/Length` that is no length, or that no `endstream` confirms, is reported as any stream's, and one an `endstream` past the one the chain stopped at confirms is reported with the rows the chain left unread. Where the section's own row for that `/Length` lies among the rows the data holds, and gives a length an `endstream` confirms, the data is taken again to it before the rows are read, and nothing is reported; nor is anything of a `/Length` a guard cut or left out of the index | +| `stream.truncated` | A stream has no `endstream` before the next object or the end of the file, and its object's `endobj` ends its data — following the declared length, which is kept, or ending the object before that stop —; or neither, no declared length ends before the end of the file, and its data runs to it. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case | | `stream.self-reference` | An object stream's dictionary names an object the stream holds — as its `/Type`, `/Length`, `/N`, `/First`, a filter or a parameter —: that object reads as null while the stream is decoded, and the stream is decoded without it | | `object-stream.member-moved` | An object is in the object stream its entry names, at another index than the entry gives, and was read where the stream's header lists it. Reported where the stream's data starts, the object, the stream and both indexes in the message | -| `object-stream.unreadable` | What an object stream — a stream an entry of the index names as one — says of itself cannot be believed: it is not of `/Type /ObjStm`, its `/N` is absent or no non-negative integer, or, when it declares objects, its `/First` is, or lies past its decoded data, or its header ends or breaks before listing as many objects as `/N` declares — a pair that is not two integers, that gives a number no object can have or an offset no member can start at, whose offset is not past the one before it, or whose member would start at or past the end of the decoded data, unless a guard cut it. The header is read as far as it goes, whatever its `/N` claims: the objects listed before the fault are read, the others cannot be read from it. Reported where the stream's data starts, the stream and its fault in the message | +| `object-stream.unreadable` | What an object stream — a stream an entry of the index names as one — says of itself cannot be believed: it is not of `/Type /ObjStm`, its `/N` is absent or no non-negative integer, or, when it declares objects, its `/First` is, or lies past its decoded data, or its header ends or breaks before listing as many objects as `/N` declares — a pair that is not two integers, that gives a number no object can have or an offset no member can start at, whose offset is not past the one before it, or whose member would start at or past the end of the decoded data, unless a guard cut it. The header is read as far as it goes, whatever its `/N` claims: the objects listed before the fault are read, the others cannot be read from it. Reported where the stream's data starts, the stream and its fault in the message. An object stream whose `/Filter`, `/DecodeParms` or a value of a step's parameters names an object a guard cut, or left out of the index, is left encoded and not reported: its objects are the guard's, unknown until it is lifted | | `object-stream.member-stream` | An object an object stream holds is written as a stream, or holds one, which ISO 32000-1 (7.5.7) forbids: it reads as null, and nothing of it is decoded. Reported once for each object and each definition of its object stream, where the stream's data starts, the object, the stream and the byte of its decoded data where the object starts in the message | | `syntax.unexpected-token` | A token stood where the syntax does not allow it: where a value or a dictionary key was expected, after a key that has no value, or closing an array with a dictionary's end; it was read as null, skipped, or it ended what it stood in | -| `syntax.truncated-object` | An object ended before it was whole: the file, or an object stream's decoded data, ended in the middle of it — a value missing, or an array, a dictionary or a string it opened never closed —, or an `endobj` stood where a value or a key should be, inside an array or a dictionary it opened. What was read is kept: a string takes the bytes to the end of the data, and an `endobj` ends every container it finds open — the object's own, or, when the object lost it too, the next object's, the container then taking that object. Reported once, for the innermost construct left open, where it opens — where the value would start, when none is open; past the depth the reader follows, the container it skipped, which holds what it did not follow —, the constructs around it counted in the message; inside an object an object stream holds, where the stream's data starts, the member and the byte in the message. A member is read to the next member's start, the last to the end of the decoded data: running into either, it is cut short there, and the message says which. The edge of a window the reader grows is no end of the data, and a cut a guard made is the guard's — a member read past `MaxObjectLength` is `limit.object`'s | +| `syntax.truncated-object` | An object ended before it was whole: the file, or an object stream's decoded data, ended in the middle of it — a value missing, or an array, a dictionary or a string it opened never closed —, or an `endobj`, or the next object's header, stood where an element, a key or a value should be, inside an array or a dictionary it opened. What was read is kept: a string takes the bytes to the end of the data, and an `endobj` ends every container it finds open, as does the next object's header, which an object that lost its `endobj` too meets first, and which leaves the object without its `endobj`. A header is `N G obj` as the search for a stream's `endstream` stops at one: digits alone, N from 1 to 2,147,483,647 and G from 0 to 65,535, white space alone between them — no comment —, and `obj` a whole keyword; tokens that read as one inside a container that closes after them end it there all the same, and the object's `endobj` is judged missing. An object stream's member, which no header follows, and a value standing alone, which is no container, never end at one. Reported once, for the innermost construct left open, where it opens — where the value would start, when none is open; past the depth the reader follows, the container it skipped, which holds what it did not follow —, the constructs around it counted in the message; inside an object an object stream holds, where the stream's data starts, the member and the byte in the message. A member is read to the next member's start, the last to the end of the decoded data: running into either, it is cut short there, and the message says which. The edge of a window the reader grows is no end of the data, and a cut a guard made is the guard's — a member read past `MaxObjectLength` is `limit.object`'s | | `syntax.depth-exceeded` | Nesting went deeper than the reader will follow | | `syntax.number-out-of-range` | A number is beyond what a real can hold — its magnitude rounds past the largest a double holds, about 1.8 × 10³⁰⁸ —: it was read as null, so a dictionary holds no entry for its key, and an array holds a null in its place. Reported where the number starts — where its object stream's data starts, for a number inside an object one holds —, quoted in the message. An integer past the range of a `long` is no fault of its own: it reads as the real nearest to it | | `syntax.hex-string-invalid` | A hexadecimal string holds bytes that are neither hexadecimal digits nor white space, the only bytes ISO 32000-1 (7.3.4.3) lets it hold — a `%` among them, which is no comment there —: they were skipped, as pdf.js and pdfium skip them, where PDFBox stops at the first, MuPDF and poppler read each as the digit 0, and qpdf and pypdf lose the string. Reported once for each string, where its first such byte lies, how many there are in the message; inside an object an object stream holds, where the stream's data starts, the member and the byte in the message | @@ -122,16 +122,17 @@ Codes are stable: they are part of the public contract, because callers filter o | `trailer.root-recovered` | The trailer's `/Root` does not lead to a document catalog, and the catalog was found among the file's objects — those its index holds when the index is sound, a rebuilt index's otherwise | | `filter.failed` | A filter's data is damaged: Flate data of which nothing decoded before the byte the fault lies in is left as the filters before it decoded it, and the filters after it are not tried; otherwise what decoded is kept, up to that byte or to the end — nothing, for LZW data whose first code is undefined, and the filters after it decode that —, and a repair when nothing was lost. ASCII85, ASCIIHex, RunLength and LZW data that breaks its format — a byte that encodes nothing, an end marker missing or out of place — is decoded as other readers decode it (*Damaged stream data*, below) | | `filter.checksum-mismatch` | A Flate stream decoded to its end, but its zlib checksum disagrees with what it decoded to: all of it was kept, and some of it may be wrong | -| `filter.unsupported` | The file names a filter the library does not implement, or an element of `/Filter` names no filter at all, or `/Filter` is neither a name nor an array; decoding stopped there, with what the filters before it decoded. A `/Filter` that is null is one the stream does not have, and is not reported | +| `filter.unsupported` | The file names a filter the library does not implement, or an element of `/Filter` names no filter at all, or `/Filter` is neither a name nor an array; decoding stopped there, with what the filters before it decoded. A `/Filter` that is null is one the stream does not have, and is not reported. Nor is a `/Filter`, an element of it, a `/DecodeParms` or a value of a step's parameters that names an object a guard cut, or left out of the index — directly or through a chain of references —: what the cut left of it, such as a name it parted, is the guard's, and decoding ends there, as it does at the element past `MaxFilterChainLength`, with what the filters before it decoded; nothing more is reported, the guard having been reported where it was reached | | `limit.decoded-stream` | A filter decodes to more than `MaxDecodedStreamLength`, or the filters before the last pass along more than it together; decoding stopped there, and the stream comes back as the filters after the cut decode what was kept within the bound | +| `limit.stream-length` | A stream's encoded data, as the file holds it, is longer than `MaxStreamLength`, which bounds what is read of it at once: the data within the bound was read, decoded as data the reader cut — what the filters lose past the cut is no damage — or handed back as it is where decoding hands data back undecoded, as with no filter, an image filter or a filter the library does not know. `Data.Length` and `RawLength` stay the length the file holds. Reported once for each stream, where its data starts, when its data is first read — by `GetRawBytes` or by decoding —, in the document's diagnostics whatever diagnostics `Decode` is given | | `limit.object` | An object is longer than `MaxObjectLength`, its stream data aside; the part within it was parsed. An object an object stream holds is bounded alike, and reported once for each definition of its object stream, where that stream's data starts, the object, the stream and the byte of its decoded data where the object starts in the message | | `limit.xref-section-length` | A classic cross-reference table is longer than `MaxXRefSectionLength`; the entries within it were read, and its trailer past them; the objects the rows past them place read as null, and the chain is not followed further | | `limit.xref-section-count` | The chain of cross-reference sections is longer than `MaxXRefSectionCount`; the newest were read, and what only the older ones index reads as null. Reported where the chain goes on, or, when that lies outside the file, at the section whose `/Prev` names it, the offset in the message | -| `limit.trailer` | A trailer, or a cross-reference stream's dictionary, is longer than `MaxTrailerLength`; the part within it was parsed, and the chain ends there: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, what only the older sections index reading as null | +| `limit.trailer` | A trailer, or a cross-reference stream's dictionary — never that stream's data —, is longer than `MaxTrailerLength`; the part within it was parsed, and the chain ends there: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, what only the older sections index reading as null | | `limit.index-entries` | The index holds `MaxIndexEntries` object numbers — those in use, free or refused — and met another: the numbers it holds were indexed, and an object past them reads as null. Reported once a document, where the first number it had no room for was met: the section that gave it, the object header or object stream a rebuild read, or the chain's newest section for an entry the chain gave that a rebuild could not keep | | `limit.filter-chain` | A stream's `/Filter` array names more filters than `MaxFilterChainLength`, `/Crypt` among them; decoding stopped before the first past them, with what the filters within the bound decoded, and the elements past them were not read. A cross-reference stream or an object stream it ends is read as cut before its data: its rows or its objects are unknown | -The seven `limit.*` codes report the reader's own limits, not faults of the file; each message names the property of +The eight `limit.*` codes report the reader's own limits, not faults of the file; each message names the property of `PdfReaderLimits` that lifts it. They are listed with their defaults in [Reader limits](reader-limits.md). ### Faults of the syntax @@ -146,6 +147,7 @@ start, so no fault is two objects'. ```text Warning syntax.truncated-object at 328: The file ended inside a literal string, which takes the 6,575 bytes from where it opens to that end; the 2 arrays or dictionaries around it were never closed either. Warning syntax.truncated-object at 381: An endobj ended the object inside an array, which was never closed. +Warning syntax.truncated-object at 507: An object header ended the object inside a dictionary, which was never closed, before the value of its last key. Warning syntax.hex-string-invalid at 1204: A hexadecimal string holds 5 bytes that are neither hexadecimal digits nor white space, the first here; they were skipped. Warning syntax.name-escape-invalid at 96: A name holds a number sign that two hexadecimal digits do not follow, kept as the byte it is: the name reads as /A#23zz. Warning syntax.key-repeated at 412: The dictionary gives the key /Filter more than once, written here with #xx escapes; the last value given is kept. @@ -153,17 +155,37 @@ Warning syntax.key-repeated at 412: The dictionary gives the key /Filter more th ### Streams whose length is wrong -A stream whose declared length the file cannot hold is reported as `stream.truncated` when the file ends inside its -data, and as `stream.length-invalid` when its `endstream` comes first. Each stream is reported once, however often it -is parsed again — after the cache let it go, or the index was rebuilt. How the reader finds where such a stream ends -is explained in [Diagnostics](../concepts/diagnostics.md#where-a-stream-ends). +A declared length is confirmed by an `endstream` after it, white space and comments between, however many. Any other +stream — its length unconfirmed, or none to confirm — is searched once for the first `endstream` after the start of its +data, up to the next object: the first object header the file's bytes hold after that start, or the next object the +file's index places, whichever is nearer. Each case is reported where the data starts, and each stream once, however +often it is parsed again — after the cache let it go, or the index was rebuilt. How the reader finds where such a +stream ends, and why, is explained in [Diagnostics](../concepts/diagnostics.md#where-a-stream-ends). + +| What follows the data | Code | The data taken | +|---|---|---| +| An `endstream` after the declared length | None | The declared length | +| No such `endstream`, or no length to confirm; an `endstream` before the next object or the end of the file | `stream.length-invalid` | Up to that `endstream`, the end-of-line before it left out | +| No `endstream` before that stop; the object's `endobj` after the declared length, which ends before the stop, white space and comments between | `stream.truncated` | The declared length | +| No `endstream` before the stop; the object's own `endobj` ends the object there, only white space between them | `stream.truncated` | Up to that `endobj`, the end-of-line before it left out | +| None of the above; a declared length that ends before the stop | `stream.length-invalid` | The declared length; whether the object ends with its `endobj` is not judged | +| None of the above; no declared length that ends before the stop | `stream.length-invalid` at the next object, `stream.truncated` at the end of the file | Up to the stop; the object lacks its `endobj`, unless what is taken holds one | +| Not searched: the document's searches for `endstream` have read as much of the file as they may, and the bytes read with the object hold neither an `endstream` nor an object header | `stream.length-invalid`, the message saying so | The declared length, when it ends before the stop; otherwise up to the stop, the file past those bytes not searched for a header | + +A `/Length` that names an object one of the reader's limits cut — directly or through a chain of references — gives no +length the file wrote: what the cut left of it, null or an integer it parted, is the guard's, which reported it. An +integer read before the cut is only a place to look for `endstream` at — the data ends there when one follows it —, +never kept as a length; the data is otherwise found by its `endstream`, as for a stream with no length, and nothing is +said of the length. A missing `endstream` is still the file's, and is reported, the message naming the object the guard +cut. A `/Length` that names an object a guard left out of the index is not judged either, and nothing is reported of +where its stream ends. ```text Warning stream.length-invalid at 5501: The stream declared 19954 bytes but ended after 19952. Warning stream.length-invalid at 19901349: The stream declared 202154 bytes, and no endstream follows them before the next object, at 20103524; the declared length is kept. Warning stream.length-invalid at 316: The stream's /Length is a real number, 61.5, not a non-negative integer; its data ends after 68 bytes. Warning stream.length-invalid at 542577: The stream's /Length names object 18 0, which holds a stream, not a non-negative integer; its data ends after 5555 bytes. -Warning stream.truncated at 315: The stream has no endstream before the endobj that follows its data; the 310 bytes to the end of the file are taken as its data. +Warning stream.truncated at 315: The stream has no endstream: its object's endobj follows the 68 bytes it declared, which are taken as its data. ``` ### Object streams @@ -203,6 +225,8 @@ Warning syntax.unexpected-token at 125: A token was found where a value was expe | Flate, nothing decodes before the fault | `filter.failed`, Warning | The data as the filters before it left it — still encoded, when it is the first —, and the filters after it are not tried | | A filter decodes past `MaxDecodedStreamLength`, or the filters before the last pass along more than it together | `limit.decoded-stream`, Warning | What the filters after the cut decode of the data up to the bound | | Names more filters than `MaxFilterChainLength` | `limit.filter-chain`, Warning | The data as the filters within the bound decoded it | +| Data longer than `MaxStreamLength` | `limit.stream-length`, Warning | What the filters decode of the data up to the bound, as data the reader cut; that data as it is, when no filter decodes it | +| A `/Filter`, an element of it, a `/DecodeParms` or a value of a step's parameters names an object a guard cut, or left out of the index | None of its own: the guard's, reported where it was reached | The data as the filters before that value decoded it | The byte counts in these reports are the reporting filter's own. The bytes kept are counted before any predictor (`/DecodeParms` `/Predictor`) is applied, so `Decode` may return another length — a PNG predictor takes a byte off each @@ -214,11 +238,11 @@ diagnostics hold what decoding a stream met once, however often the stream is de or the document's, by the reader as it serves an object stream's objects again after its budget let them go or as it rebuilds the index, or by the validator —, as long as they have room for it; diagnostics you pass of your own hold every decoding's reports. A stream the reader itself cut at one of its limits is reported as that limit, not as a lost tail; a -fault in the part it did read is the file's, and is reported. That holds whoever cut it: the window the parser reads an -object through, or a filter before this one that stopped at `MaxDecodedStreamLength`. Of data the reader cut, no lost -tail, missing checksum or missing end marker is reported, a hexadecimal digit without its pair or an ASCII85 group short -of five is dropped rather than completed, and a predictor undoes the part of its last row the data holds rather than -report rows the parameters describe as longer than the data. +fault in the part it did read is the file's, and is reported. That holds whoever cut it: the reader, reading the stream's +data within `MaxStreamLength`, or its object within `MaxObjectLength`, or a filter before this one that stopped at +`MaxDecodedStreamLength`. Of data the reader cut, no lost tail, missing checksum or missing end marker is reported, a +hexadecimal digit without its pair or an ASCII85 group short of five is dropped rather than completed, and a predictor +undoes the part of its last row the data holds rather than report rows the parameters describe as longer than the data. ```text Warning filter.failed at 59534: A Flate stream ends before its data does; what decoded before the end was kept. @@ -227,6 +251,7 @@ Warning filter.failed at 2102: A Flate stream is corrupt at byte 889 of its 2062 Warning filter.checksum-mismatch at 3278: A Flate stream's checksum disagrees with the 174803 bytes its data decoded to; all were kept, and some may be wrong. Warning filter.failed at 4336336: An ASCII85 stream ends without its ~> end marker, after a group of a single digit, which encodes no byte; it was dropped, and data may have been lost. Warning filter.failed at 812: An ASCIIHex stream holds a byte that encodes nothing, at byte 5 of its 12; it was passed over. +Warning limit.stream-length at 9318: A stream's data runs past 256 MB; only what lies within it was read. Raise PdfReaderLimits.MaxStreamLength to read past it. ``` ## Exceptions diff --git a/docs/website/docs/reference/object-model.md b/docs/website/docs/reference/object-model.md index b839650e..931d9f05 100644 --- a/docs/website/docs/reference/object-model.md +++ b/docs/website/docs/reference/object-model.md @@ -21,7 +21,7 @@ references they hold. Every one derives from `PdfObject`. How a document reads t | `PdfString` | A string's bytes, text or binary | `Bytes`; `ToText()` for its text | Its text between double quotes, escaped: `"Annual report"` | | `PdfArray` | An ordered list of objects | Its items, by index | `[array of 3]` | | `PdfDictionary` | Entries, each keyed by a name | Its entries, by key | `<>`, or `<>` when its `/Type` is no name | -| `PdfStream` | A dictionary and its encoded data | `Dictionary`, `GetRawBytes()`; `Decode()` for the decoded data | `<>`, by its `/Subtype`, else its `/Type` | +| `PdfStream` | A dictionary and its encoded data, which a stream read from a document [reads each time it is asked for](#a-streams-data) | `Dictionary`, `GetRawBytes()`; `Decode()` for the decoded data | `<>`, by its `/Subtype`, else its `/Type` | | `PdfReference` | A reference to an object, such as `12 0 R` | `Id`; `Resolve()` for the object | `12 0 R` | ## What `ToString()` gives @@ -84,6 +84,29 @@ writes past what a `long` holds is read as a real, and so described with its poi The longest description is 327 characters, that of -4.9E-324, the negative of the smallest positive double. +## A stream's data + +A `PdfStream` holds its dictionary and a `PdfStreamData`. For a stream read from a document, that says where the +encoded bytes lie and how many there are, not what they are: they are read each time something asks for them, and kept +by nothing — neither the stream nor the document's object cache —, so that a stream costs its data only while it is +read or decoded, and for as long as the caller keeps what that gave. + +| Member | What it gives | What it costs | +|---|---|---| +| `Dictionary` | The stream's dictionary | Nothing: it was read with the stream | +| `RawLength`, `Data.Length` | The count of the stream's encoded bytes in the file: its `/Length` where an `endstream` confirms it, otherwise as far as the reader takes the data to run ([Where a stream ends](../concepts/diagnostics.md#where-a-stream-ends)). Data longer than `MaxStreamLength` is counted whole | Nothing: found when the stream is parsed | +| `Data.Position` | The offset in the file the data starts at | Nothing | +| `GetRawBytes()`, `Data.GetBytes()` | The encoded bytes, as the file holds them, up to `MaxStreamLength`: past it, the bytes within the bound, and `limit.stream-length` reported once for the stream | A read, each call: from a document opened from bytes or from a stream, the bytes lent where they lie, without a copy; from one opened from a path, read from the file into an array of their length | +| `Decode()` | The data the stream's filters decode, as far as they go: decoding stops at an image filter, at a filter the library does not know, and where the [reader limits](reader-limits.md) end it | A read, as `GetRawBytes()` makes, then what each filter produces; data no filter decodes comes back as it was read | + +Opened with `ThrowOnLimit`, a document's streams throw a `PdfLimitExceededException` from `GetRawBytes()` and +`Decode()` instead, each time their data runs past `MaxStreamLength`; `Decode()` throws at the other limits it reaches +too. No call's result is kept for the next: a caller that uses the data twice keeps what one call gave, rather than call +again ([Tune the reader's memory](../guides/tune-reader-memory.md#read-a-streams-data-once)). + +Data built with `PdfStreamData.FromMemory` is held by the stream, whatever its length, and handed back as it was given; +its `Position` is -1. + ## Objects from a disposed document An object keeps what it holds once its document is disposed: a dictionary's direct values, an array's items, a stream's diff --git a/docs/website/docs/reference/validation-rules.md b/docs/website/docs/reference/validation-rules.md index 0226ba58..0b420604 100644 --- a/docs/website/docs/reference/validation-rules.md +++ b/docs/website/docs/reference/validation-rules.md @@ -43,15 +43,15 @@ References are to ISO 32000-1:2008. | `file.startxref-wrong` | Error | The offset `startxref` gives holds no cross-reference section: the index is out of reach, and is rebuilt by scanning. White space before the section is `xref.offset-imprecise`'s; a section that is there and cannot be read is `xref.section-malformed`'s. | The first section of the chain | | `file.trailer-missing` | Error | A cross-reference table's rows are not followed by the `trailer` keyword: they run into a dictionary, or the file ends. qpdf, as the reader, then rebuilds the index. | Each classic table of the chain | | `file.trailer-malformed` | Error | A trailer is not a well-formed dictionary: the `trailer` keyword is not followed by one, or the reader read it despite syntax errors — a key that is not a name, a dictionary or a string left unclosed, a hexadecimal string holding bytes that are neither digits nor white space, a name whose number sign is no escape, a key given twice —, so an entry may have been lost with the syntax, or read otherwise than its writer meant. A cross-reference stream's dictionary is held to the same. | Each trailer of the chain, as the reader parsed it | -| `file.root-invalid` | Error | The trailer's `/Root` is missing or does not lead to a document catalog (Table 15), a value that is not a reference included — unless it is a catalog written in the trailer itself, which the reader takes as it is ([#111]). The reader then looks for an object of `/Type /Catalog` among those the index holds — and rebuilds the index only when none is one —: the catalog it finds is its choice, not the file's. Not judged when the trailer could not be read at all, nor when the index was rebuilt as the document opened, nor when one of the reader's limits left the catalog out of the index as the document opened, which `xref.checked-in-part` says, or cut the object `/Root` names — through a chain of references too, and in an object stream whose cut leaves it unread, past the cut data or in a stream the guard made unreadable ([#259]) — or the trailer a catalog written there was read from, which the reader then takes for the catalog as it read it, looking for no other, and its `limit.*` report names ([#175]) — whatever a rebuild found since —, nor a `/Root` the trailers lack where one of the reader's limits kept a trailer from being read whole — cut, or older than where the chain stopped, at the section count or at a section whose rows a limit cut or filled the index with —: it may lie there, and the reader rebuilds nothing for it. Rows a limit cut that run on to no trailer lack one: the file's, which a rebuild answers. The catalog the finding names is the one the reader took last, in the rebuild run before the first rule if there was one. | The trailer the chain gave, merged as readers merge it, and `/Root` before the reader looked for a catalog | +| `file.root-invalid` | Error | The trailer's `/Root` is missing or does not lead to a document catalog (Table 15), a value that is not a reference included — unless it is a catalog written in the trailer itself, which the reader takes as it is ([#111]). The reader then looks for an object of `/Type /Catalog` among those the index holds — and rebuilds the index only when none is one —: the catalog it finds is its choice, not the file's. Not judged when the trailer could not be read at all, nor when the index was rebuilt as the document opened, nor when one of the reader's limits left the catalog out of the index as the document opened, which `xref.checked-in-part` says, or cut the object `/Root` names — through a chain of references too, and in an object stream whose cut leaves it unread, past the cut data or in a stream the guard made unreadable ([#259]), or in one whose filters end at a `/Filter`, a `/DecodeParms` or a parameter that leads to an object a guard cut or left out of the index, which leaves its data encoded ([#263]) — or the trailer a catalog written there was read from, which the reader then takes for the catalog as it read it, looking for no other, and its `limit.*` report names ([#175]) — whatever a rebuild found since —, nor a `/Root` the trailers lack where one of the reader's limits kept a trailer from being read whole — cut, or older than where the chain stopped, at the section count or at a section whose rows a limit cut or filled the index with —: it may lie there, and the reader rebuilds nothing for it. Rows a limit cut that run on to no trailer lack one: the file's, which a rebuild answers. The catalog the finding names is the one the reader took last, in the rebuild run before the first rule if there was one. | The trailer the chain gave, merged as readers merge it, and `/Root` before the reader looked for a catalog | | `file.size-wrong` | Warning | A section's `/Size` is missing, or is not one more than the highest object number it and the sections it updates use (Tables 15 and 17, 7.5.6, 7.5.8.4). A hybrid file's cross-reference stream meets one of Table 17's two readings or the other — one more than the highest number it and the sections it updates use, or the `/Size` of the table that names it —, and either is accepted. A `/Size` that leaves in-use objects numbered above it is `xref.object-past-size`'s. A `/Size` written as a reference, or as a real, is said to be. Not judged when the chain lost a section, or one of the reader's limits cut it. | Each section's trailer, and the highest number each section indexes | -| `file.trailer-value-wrong` | Warning | A value a section's trailer gives the reader to read the chain is written as the specification does not allow, and read all the same, as other readers read it: a reference where it must be direct, read where a section read before it places the object; a real with no fractional part where it must be an integer, read as the integer it equals ([#182], [#215]). A cross-reference stream's `/Type`, `/Index`, `/Prev` and `/W`, the elements of `/Index` and `/W`, and its `/Filter` and `/DecodeParms` are direct (7.5.8.2); in a file that declares PDF 2.0, so are its `/Length` and the elements and entries of its `/Filter` and `/DecodeParms` (an erratum of ISO 32000-2, pdf-issues #246). A classic trailer's `/Prev` is direct (Table 15); an `/XRefStm` may be indirect (Table 19). `/W`, `/Index`, `/Prev`, `/XRefStm`, a stream's `/Length` and its `/Predictor`, `/Colors`, `/BitsPerComponent`, `/Columns` and `/EarlyChange` are integers. One finding per value, which says both faults of a reference that leads to a real. A value the rows are read with that the reader could not read so is the section's fault — `xref.section-malformed`, `xref.section-not-found` —, and `/Size` is `file.size-wrong`'s; a `/Type`, or a `/Length` read once the chain is, is reported here whether or not it could be read. | Each section's trailer, as the file wrote it, and the objects its references name, through the index the reader settled on before the first rule | +| `file.trailer-value-wrong` | Warning | A value a section's trailer gives the reader to read the chain is written as the specification does not allow, and read all the same, as other readers read it: a reference where it must be direct, read where a section read before it places the object; a real with no fractional part where it must be an integer, read as the integer it equals ([#182], [#215]). A cross-reference stream's `/Type`, `/Index`, `/Prev` and `/W`, the elements of `/Index` and `/W`, and its `/Filter` and `/DecodeParms` are direct (7.5.8.2); in a file that declares PDF 2.0, so are its `/Length` and the elements and entries of its `/Filter` and `/DecodeParms` (an erratum of ISO 32000-2, pdf-issues #246). A classic trailer's `/Prev` is direct (Table 15); an `/XRefStm` may be indirect (Table 19). `/W`, `/Index`, `/Prev`, `/XRefStm`, a stream's `/Length` and its `/Predictor`, `/Colors`, `/BitsPerComponent`, `/Columns` and `/EarlyChange` are integers. One finding per value, which says both faults of a reference that leads to a real. A value the rows are read with that the reader could not read so is the section's fault — `xref.section-malformed`, `xref.section-not-found` —, unless it leads to an object one of the reader's limits cut, which leaves the section to `xref.checked-in-part` ([#263]), and `/Size` is `file.size-wrong`'s; a `/Type`, or a `/Length` read once the chain is, is reported here whether or not it could be read. | Each section's trailer, as the file wrote it, and the objects its references name, through the index the reader settled on before the first rule | ### Cross-references | Rule | Severity | Meaning | Reads | |---|---|---|---| -| `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]). A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. | Each section of the chain | +| `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]); or a cross-reference stream whose `/Length` is such a reference, and whose data no `endstream` ends before the next object the sections read so far place, or the end of the file — or was not searched for one, the document's searches for `endstream` having read as much of the file as they may ([#220]). Rows that spell `endstream` end such a stream's data there, and the section lacks the rows past them, unless the section's own row for that `/Length`, among the rows before them, places an object that gives a longer length an `endstream` confirms: the data is then taken to it. A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. Nor is a cross-reference stream whose value the rows are read with leads to an object a limit cut: the section counts as cut there, the chain ends at it, nothing is rebuilt, and `xref.checked-in-part` says so ([#263]). | Each section of the chain | | `xref.section-not-found` | Error | A section a `/Prev` or an `/XRefStm` names is neither there nor within 512 bytes of it, or what names it is not an offset — tiff2pdf writes `/Prev 576066 0 R`. The objects only it indexes are missing from the index. The section `startxref` names is `file.startxref-wrong`'s. | Each section of the chain | | `xref.section-shifted` | Warning | A section a `/Prev` or an `/XRefStm` names starts a few bytes from where it is named, and the reader found it there. | Each section of the chain | | `xref.chain-loop` | Error | The chain names a section it has already read, or an offset it has already named where it read none, in the file or outside it: the reader stops there, and the section the chain should have gone on to is unknown, what only it indexed found, if at all, by rebuilding the index — as for `xref.section-not-found`. | The chain | @@ -59,7 +59,7 @@ References are to ISO 32000-1:2008. | `xref.entry-shifted` | Warning | An in-use entry places its object a few bytes from where it is — the object's own header confirming it —, or at another index of its object stream than the one its header gives. One finding per entry, located at the object alone when the entry's offset lies outside the file, a load having found the object near it ([#175]). | As `xref.entry-broken` | | `xref.generation-mismatch` | Warning | An in-use entry gives a generation other than the one its object is written with. The references and the object's header agree against the entry, and the reader reads the object; qpdf reads a reference to it as a reference to nothing. | Each object's header | | `xref.object-stream-circular` | Error | An object stream needs, to be read, an object only reading it can give: its `/Length`, `/Type`, `/Filter`, `/DecodeParms`, `/N` or `/First` names an object it holds — directly, through objects written in the file, or through another object stream whose own keys need it in turn (7.5.7) —, a name or a number through a chain of references alone, as far as a reference resolves — a `/Length` one object further, which the parse reads before it resolves it —, and a filter and its parameters whole, every element and entry of their arrays and dictionaries and of the objects they name — more than decoding may read, and nothing of a parameter that is a stream, which [#277] will align with what decoding reads. The reader resolves a stream's `/Length` as it parses it, then its `/Type`, and nothing else of a stream not of `/Type /ObjStm` ([#274]): of such a stream, its `/Type` alone is followed, a loop through it being why the stream is not taken for an object stream; a loop through its `/Length` leaves its objects lost to its type all the same, `xref.object-stream-broken`'s. The reader decodes the stream without that value, as qpdf reports a loop, and cannot vouch for what it decoded ([#51]). One finding per stream; `xref.object-stream-broken` says nothing more of it. | Each object stream's dictionary and the objects its decoding keys name; nothing is decoded | -| `xref.object-stream-broken` | Error | An object stream the index places objects in is not in the index, is not an object stream — not of `/Type /ObjStm`, which the reader reads no object from either ([#274]) —, or does not decode to the header its `/N` and `/First` describe: every object placed in it is lost. A header breaks at a pair that is not two integers, that names a number no object can have or an offset no object can start at, whose offset is not past the one before it, or whose object would start at or past the end of the decoded data, unless a guard cut it (7.5.7); when it breaks after listing some objects, the reader reads those, and the objects lost are those the index places in the stream and the header does not list before the break, which the message counts — none, and no finding, when the header lists every one ([#275]). One finding per stream. An object stream whose own entry is broken is `xref.entry-broken`'s, and one the chain's index lacks once a reader limit stopped it short `xref.checked-in-part`'s, though a rebuild found the stream since. | Each object stream once, without keeping it — once more when the reader rebuilt its index, the chain's index places some of the objects in it, and only the rebuilt index others | +| `xref.object-stream-broken` | Error | An object stream the index places objects in is not in the index, is not an object stream — not of `/Type /ObjStm`, which the reader reads no object from either ([#274]) —, or does not decode to the header its `/N` and `/First` describe: every object placed in it is lost. A header breaks at a pair that is not two integers, that names a number no object can have or an offset no object can start at, whose offset is not past the one before it, or whose object would start at or past the end of the decoded data, unless a guard cut it (7.5.7); when it breaks after listing some objects, the reader reads those, and the objects lost are those the index places in the stream and the header does not list before the break, which the message counts — none, and no finding, when the header lists every one ([#275]). One finding per stream. An object stream whose own entry is broken is `xref.entry-broken`'s, and one the chain's index lacks once a reader limit stopped it short `xref.checked-in-part`'s, though a rebuild found the stream since — as is one whose decoding a guard ended, its data left encoded, at the filter past `MaxFilterChainLength` or at a `/Filter`, a `/DecodeParms` or a parameter that leads to an object a guard cut or left out of the index ([#263]). | Each object stream once, without keeping it — once more when the reader rebuilt its index, the chain's index places some of the objects in it, and only the rebuilt index others | | `xref.object-stream-value-wrong` | Warning | An object stream gives its `/N`, `/First` or `/Length` as a real number with no fractional part, where the specification asks for an integer (Tables 5 and 16). The reader reads it as the integer it equals, as MuPDF, PDFBox and pdf.js do; qpdf and poppler lose the stream's objects ([#215]). One finding per stream and key. A value with a fractional part is another fault: a `/N` or a `/First` leaves the stream unreadable, `xref.object-stream-broken`'s, and a `/Length` is recovered through the stream's `endstream`, which the reader reports (`stream.length-invalid`). | As `xref.object-stream-circular` | | `xref.offset-imprecise` | Warning | Offsets name the white space before what they designate rather than its first byte (7.5.4, 7.5.5): Microsoft Print to PDF names the line feed before each object, and some tools the one before `xref`. Every reader skips the white space. One finding for the file, with how many offsets and the first of them. | The sections of the chain, and each object's header | | `xref.object-past-size` | Warning | In-use objects are numbered above the trailer's `/Size`, which Table 15 makes a conforming reader ignore ("shall be ignored and defined to be missing"). The entries, the objects' headers and the references agree against `/Size`, and the reader reads those objects, as qpdf does while warning. One finding for the file, with how many objects and the first of them; `file.size-wrong` then says nothing more of that `/Size`. | The index and the trailer's `/Size` | @@ -87,7 +87,7 @@ it is checked: the exception above judges the index the reader rebuilt, and the | Rule | Severity | Meaning | Reads | |---|---|---|---| | `object.reference-missing` | Warning | An object reachable from the trailer refers to an object the file does not define — none in its index, a free entry, or object 0, which heads the free list and is never in use (7.5.4) —, which is null (7.3.10): the reader reads it so, rebuilding nothing, and what the reference was to give is lost. One finding per object that holds such references, naming the first. An object one of the reader's limits left out of the index is unknown rather than missing, and is `xref.checked-in-part`'s. An object the index holds and the reader cannot produce is `xref.entry-broken`'s or `xref.object-stream-broken`'s — but one a rebuild serves from another object stream than the chain's index places it in, or from another definition of that stream, which does not list it or holds it written as a stream, and so loses, which no finding names yet ([#276]) —, a kid of the page tree `page-tree.kid-invalid`'s — one in a node's `/Kids`, an array of kids of its own or a root written in the catalog the walk read whole, not what an array written among the kids holds, which the walk does not read —, and a trailer's `/Root` that names nothing `file.root-invalid`'s. | Every object reachable from the trailer, once each | -| `object.endobj-missing` | Warning | An object reachable from the trailer does not end with `endobj` (7.3.10): another token, or the end of the file, follows its value. The reader reads the value as far as it goes, as qpdf does while reporting it. An empty object, `2 0 obj endobj`, has its `endobj`. After a stream whose data runs past the 8 KB window the reader first reads through, the file is asked for the bytes after its `endstream` since [#55]. More than a few dozen bytes of white space or comment before the next token leave what follows unseen, and a stream whose declared length the reader kept for want of an `endstream`, or whose data an `endobj` follows without one, leaves where the object ends unknown: neither is judged. | What follows each reachable object's value, as the reader read it | +| `object.endobj-missing` | Warning | An object reachable from the trailer does not end with `endobj` (7.3.10): another token, or the end of the file, follows its value. The reader reads the value as far as it goes, as qpdf does while reporting it. An empty object, `2 0 obj endobj`, has its `endobj`. An object whose array or dictionary was never closed lacks it where the next object's header — `N G obj` in digits, white space alone between them, an object number from 1 to 2,147,483,647 and a generation up to 65,535 —, met where an element, a key or a value is expected, ends every container left open ([#230]); an array or a dictionary that holds such a header and is closed after it ends there all the same, its `endobj` judged missing, which none of the corpus's 344,361 objects does. After a stream whose data runs past the 8 KB window the reader first reads through, the file is asked for what follows its `endstream` ([#55]), read through white space and comments however long ([#174]); after an `endstream` inside that window, white space that runs past the window's edge leaves what follows unseen, and not judged. A stream with no `endstream` has its `endobj` when the object's `endobj` follows the length it declared, or ends its data with nothing but white space after it before the next object or the end of the file; one whose data runs on to the next object, or to the end of the file, lacks it. Where the object ends is unknown, and is not judged, after a stream whose declared length the reader kept with neither an `endstream` nor an `endobj` after it, one whose data, run on to the next object or the end of the file, holds an `endobj` that may be its object's own, and one whose end lies past what the reader reads first once the document's searches for `endstream` have read as much of the file as they may. | What follows each reachable object's value, as the reader read it | | `object.name-null-character` | Warning | An object reachable from the trailer holds a name with a null character, written `#00`, which a name cannot contain (7.3.5). The reader keeps the name; qpdf refuses it and reads dictionaries without the keys so named. One finding per object, naming the first. | Every name of every reachable object | | `object.key-missing` | Warning | An object reachable from the trailer lacks a key the [Arlington PDF Model](#object-shapes-from-the-arlington-pdf-model) requires of its type — absent, or `null`, which in a dictionary is the same (7.3.7): a key the model requires once the version the file declares has it, or one it requires in some versions only, such as a form XObject's `/Name` in PDF 1.0. An inheritable key is looked for up the `/Parent` chain, and a field's `/DA` in the interactive form's too; a key only an extension of ISO 32000 defines is never required. An array shorter than the elements it requires lacks each required element it does not hold, one finding per element. [One finding per type and key](#object-shapes-from-the-arlington-pdf-model), at the first object, with how many lack it. | Every object reachable from the trailer, typed along the model's links, once each | | `object.value-type-wrong` | Warning | A value is of a type the Arlington PDF Model does not allow for its key, judged by the value's class: a real, even `400.0`, where an integer belongs; a string where a name does; a stream where a dictionary does, or the reverse, which is reported on the object that holds it and not walked into. A `null` written in an array is judged like any other value; one reached through a reference is not judged. One finding per type and key. | As `object.key-missing` | @@ -191,8 +191,8 @@ element — weighing most, and **a tie leaves the object unchecked** rather than not decode, or could not decrypt before M16, or left out of the index by a reader limit —, is present, and is not checked: that is `object.reference-missing`'s or the cross-reference rules' business. An object a reader limit cut — at `MaxObjectLength`, or a member of an object - stream whose data `MaxDecodedStreamLength` cut — is not judged either, nor what is written inside it: what it lacks - may lie past the limit. An ancestor so cut, or one a guard left out of the index, gives any key an object would inherit + stream whose data `MaxStreamLength` or `MaxDecodedStreamLength` cut, or whose decoding a guard ended ([#263]) — is not + judged either, nor what is written inside it: what it lacks may lie past the limit. An ancestor so cut, or one a guard left out of the index, gives any key an object would inherit from it. - Keys are taken in ordinal order and elements in order, so that two validations give the same findings in the same order. @@ -304,9 +304,10 @@ Some faults the structural profile does not report, each for a reason: from a stray dictionary typed `/Page`. - **An array, a dictionary or a string still open at the end of the data** — the reader takes it as it stands and reports it as `syntax.truncated-object`, in the document's diagnostics ([#119]); `object.endobj-missing` reports the - object that runs to the end of the file, as it reports any other object without its `endobj` — but for one that lost - its container's closing too, whose container the next object's `endobj` ends: the reader reads that `endobj` as the - object's, and only `syntax.truncated-object` tells ([#230]). + object that runs to the end of the file, as it reports any other object without its `endobj`, and the one whose + containers the next object's header ended ([#230]). A container an `endobj` ends is `syntax.truncated-object`'s alone, + that `endobj` being the object's — or the next object's, when the object lost its own and that object's header holds + a comment, which the header rule does not take for one: the container then runs on to that `endobj`. The object-shape rules read part of the Arlington PDF Model, and leave the rest silent until a rule of its own is worth its noise: @@ -357,15 +358,18 @@ PDF/UA profiles of the `AdCodicem.Pdf.Conformance` package (M20) take families o [#128]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/128 [#129]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/129 [#161]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/161 +[#174]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/174 [#175]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/175 [#182]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/182 [#215]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/215 [#219]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/219 +[#220]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/220 [#221]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/221 [#230]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/230 [#237]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/237 [#256]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/256 [#259]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/259 +[#263]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/263 [#274]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/274 [#275]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/275 [#276]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/276 diff --git a/src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs b/src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs index a16805c7..da088a0e 100644 --- a/src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs +++ b/src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs @@ -35,7 +35,9 @@ public static class PdfDiagnosticCodes /// A cross-reference section the chain names, through /Prev or /XRefStm, is where it is named but /// cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects /// past 2,147,483,647, a cross-reference stream whose /W, /Index, /Size or data cannot give - /// rows —, and nothing near it can be read in its place. The rows read before the + /// rows, or whose /Length no section read before it places and whose data no endstream ends before the + /// next object or the end of the file —, and nothing near it can be read in its place. A value that leads to an object + /// one of the reader's limits cut is the limit's: the chain ends there, and nothing is reported of the section. The rows read before the /// fault, if any, were kept; the objects only the rest indexes are missing from the index, which is rebuilt by /// scanning the file when one of them is asked for. The report is placed at the section, the fault in its message. /// @@ -59,13 +61,15 @@ public static class PdfDiagnosticCodes /// A stream's /Length is not where its data ends: its endstream lies elsewhere and ends the data, the /// /Length gives no length the reader can take and the endstream ends the data, or no endstream /// follows the declared length — none before the next object or the end of the file, or none looked for once the - /// document's searches read as much of the file as they may — and that length is kept. + /// document's searches read as much of the file as they may — and that length is kept, or the data runs to the next + /// object. A /Length that leads to an object one of the reader's limits cut, or left out of the index, is not + /// judged: the limit's report says why. /// public const string StreamLengthInvalid = "stream.length-invalid"; /// - /// A stream has no endstream before the end of the file, or before the endobj that follows its data: - /// its data runs to the end of the file. + /// A stream has no endstream: its object's endobj ends its data — after its declared length, or where the + /// object ends —, or its data runs to the end of the file. /// public const string StreamTruncated = "stream.truncated"; diff --git a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs index 23af747e..b036c815 100644 --- a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs +++ b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs @@ -105,9 +105,9 @@ private static ReadOnlyMemory DecodeChain( { leftEncoded = false; - // Whether the data a filter is given is data the reader cut at one of its limits: the parser's window cut the - // stream, or a filter before it stopped at MaxDecodedStreamLength. Every filter after a cut is given cut data, - // and the cut is reported once, by the guard that made it. + // Whether the data a filter is given is data the reader cut at one of its limits: MaxStreamLength, or the window a + // guard stopped at, cut the stream's data, or a filter before it stopped at MaxDecodedStreamLength. Every filter + // after a cut is given cut data, and the cut is reported once, by the guard that made it. var cut = stream.Data.CutByGuard; var limitReported = false; diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 6f144071..2686102f 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -4823,8 +4823,9 @@ private void RecordStreamLength(int number, int found, PdfObject parsed, StreamL } /// - /// Marks a stream whose data reaches the edge of the window a guard stopped at: the reader cut it there, - /// where its endstream was still to be found, and its lost tail is the guard's, not the file's. + /// Marks a stream whose data reaches the edge of the window a guard stopped at — its dictionary ran to that edge, the + /// data no search can be anchored at past it —: the reader cut it there, and its lost tail is the guard's, not the + /// file's. A stream whose data starts within the window is found by its endstream, which no window bounds. /// private static void MarkCutByGuard(PdfObject parsed, long windowEnd) { @@ -6038,12 +6039,8 @@ private void PreserveChainIndex() } /// - /// A stream's data in the file, read the first time something asks for it, and refused, read or not, once the - /// document is disposed. - /// - /// - /// A stream's data in the file: where it lies and how long it is, read each time it is asked for, and kept by - /// nothing (#170). + /// A stream's data in the file: where it lies and how long it is, read each time it is asked for, kept by nothing + /// (#170), and refused once the document is disposed. /// /// /// Streams whose data share a stretch of the file — overlapping objects —, or the many streams of a document read once diff --git a/src/AdCodicem.Pdf/Objects/PdfStream.cs b/src/AdCodicem.Pdf/Objects/PdfStream.cs index 6e93548e..8d832e13 100644 --- a/src/AdCodicem.Pdf/Objects/PdfStream.cs +++ b/src/AdCodicem.Pdf/Objects/PdfStream.cs @@ -30,6 +30,17 @@ public PdfStream(PdfDictionary dictionary, PdfStreamData data) public int RawLength => Data.Length; /// Returns the encoded bytes, exactly as they appear in the file. + /// + /// The bytes are read from the document's source each time they are asked for, and kept by nothing: a document held in + /// memory lends them where they lie, and any other source is read again. No more than + /// bytes are read: past it, what lies within it comes back, and + /// is reported once for the stream. + /// + /// The stream's document is disposed. + /// + /// The data runs past , and the document was opened with + /// . + /// public ReadOnlyMemory GetRawBytes() => Data.GetBytes(); /// diff --git a/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs b/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs index 69f252f0..aaeed4a1 100644 --- a/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs +++ b/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs @@ -53,7 +53,8 @@ public static class PdfStreamDecodingExtensions /// /// /// - /// Decoding reaches one of its document's guards — the data decodes past + /// Decoding reaches one of its document's guards — the encoded data runs past + /// , the data decodes past /// , or decoding reaches the filter past /// —, and the document was opened with /// . diff --git a/src/AdCodicem.Pdf/Validation/Rules/EndObjMissingRule.cs b/src/AdCodicem.Pdf/Validation/Rules/EndObjMissingRule.cs index 7bdf16c3..b2f2c651 100644 --- a/src/AdCodicem.Pdf/Validation/Rules/EndObjMissingRule.cs +++ b/src/AdCodicem.Pdf/Validation/Rules/EndObjMissingRule.cs @@ -11,9 +11,9 @@ namespace AdCodicem.Pdf.Validation.Rules; /// was evidently meant, a warning (ADR 45). An empty object, 2 0 obj endobj, has its endobj. /// /// -/// The objects checked are those the reader read, reached from the trailer; what follows a stream whose data runs past -/// the window the reader first reads through is not seen until #55 checks such a stream's length, nor what follows an -/// object one of the reader's limits cut. An object stream's members have no endobj of their own. +/// The objects checked are those the reader read, reached from the trailer; what follows an object one of the reader's +/// limits cut is not seen, nor what follows a stream whose declared length no endstream nor endobj follows, +/// kept with its object's end unknown. An object stream's members have no endobj of their own. /// /// internal sealed class EndObjMissingRule : IValidationRule From f57375e7b673828c10b4179c1788b79cd15fb488 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 07:53:29 +0000 Subject: [PATCH 012/146] fix: read a stream's end the same however it is read again and wherever its window ends The review of the batch found four places where a stream's end depended on how it was read rather than on its bytes. - The search for a stream's endstream is kept once per stream, but whether the object's endobj follows the declared length was kept with it. A rebuild that replaced the /Length then read the stream with the old answer. It is now read again whenever the declared end changes, and only then, so a second reading of an unchanged stream reads nothing more. - An endstream the window held, with the window ending before what follows it, left the object's endobj unseen. Past the window, what follows is read on in the file. It is now read on in both cases, so where the window ends decides nothing. - A file that ends on the stream keyword, or on its carriage return, read the stream as no data and said nothing: the parser took the end of the bytes for a window too short to say where the data starts. The end of the file says it: the data starts there and holds nothing, and the missing endstream is reported, as after a line feed. - Closing the reader released the searches but not the checks made past white space after a length. The states a missing endstream ends in, and the next object a fault names, are documented as the search now decides them. Refs #174 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/EndStreamState.cs | 19 +++++--- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 48 ++++++++++++++----- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 12 ++++- src/AdCodicem.Pdf/IO/StreamLengthFault.cs | 5 +- tests/AdCodicem.Pdf.Tests/ReadOrderTests.cs | 39 +++++++++++++++ .../AdCodicem.Pdf.Tests/StreamLengthTests.cs | 21 ++++++++ tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs | 32 +++++++++++++ 7 files changed, 153 insertions(+), 23 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/EndStreamState.cs b/src/AdCodicem.Pdf/IO/EndStreamState.cs index 098b9f24..78086071 100644 --- a/src/AdCodicem.Pdf/IO/EndStreamState.cs +++ b/src/AdCodicem.Pdf/IO/EndStreamState.cs @@ -8,23 +8,28 @@ internal enum EndStreamState : byte /// /// Nowhere between the start of the data and the next object after it — the one an index places, or the first header - /// the file's bytes hold, whichever is nearer —: the declared length is kept. + /// the file's bytes hold, whichever is nearer —, and no endobj of its object ends the data: the declared length + /// is kept when it ends before that object, and the data runs up to it otherwise. /// MissingBeforeNextObject, /// - /// Nowhere between the start of the data and the end of the file: the declared length is kept when the file can - /// hold it, and the data runs to the end of the file otherwise. + /// Nowhere between the start of the data and the end of the file, and no endobj of its object ends the data: + /// the declared length is kept when the file can hold it, and the data runs to the end of the file otherwise. /// MissingBeforeEndOfFile, - /// Nowhere before the endobj that follows the data, which is taken to run to the end of the file. + /// + /// Nowhere before the next object or the end of the file, and its object's endobj plays its part: the data is + /// the declared length the endobj follows, or ends where that endobj ends the object. + /// MissingBeforeEndObj, /// - /// Not looked for: the document's searches for endstream read as much of the file as they may, which only - /// searches that share a stretch of it make them do — objects that overlap, or headers the search does not take that - /// no index as written places. The declared length is kept. + /// Not looked for past the bytes the parser held: the document's searches for endstream read as much of the + /// file as they may, which only searches that share a stretch of it make them do — objects that overlap, or headers + /// the search does not take that no index as written places. The declared length is kept when it ends before the + /// next object, and the data runs up to it otherwise. /// NotSearched, } diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 2686102f..2ac0f86d 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -364,9 +364,9 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// /// What each search of the file for a stream's endstream found, by where the stream's data starts and whether /// its /Length was one the chain could not read: the file is searched once for each stream, however many times - /// it is parsed. + /// it is parsed. With it, the declared end it last said what follows of, read again only when that end changes. /// - private readonly Dictionary<(long DataStart, bool Deferred), StreamEndSearch> _endStreamSearches = []; + private readonly Dictionary<(long DataStart, bool Deferred), (StreamEndSearch Search, long DeclaredEnd)> _endStreamSearches = []; /// /// What follows each place the reader read on past white space and comments to find an endstream (#174): read @@ -1790,11 +1790,38 @@ private bool IsEndObjKeywordAt(long position) StreamEndSearch IPdfStreamDataProvider.FindEndStream( int number, long absoluteDataStart, long declaredEnd, ReadOnlySpan buffered, bool deferred) { - if (_endStreamSearches.TryGetValue((absoluteDataStart, deferred), out var known)) + var found = _endStreamSearches.TryGetValue((absoluteDataStart, deferred), out var known); + + if (found && known.DeclaredEnd == declaredEnd) { - return known; + return known.Search; + } + + var search = found ? known.Search : SearchEndStreamOnce(number, absoluteDataStart, buffered, deferred); + + // Whether the object's endobj follows the declared length depends on the length the stream is read with now, which a + // rebuild can change — the /Length an object another definition replaces —: it is read again when that length does, + // and the search, whose result the bytes and the index as written alone decide, is not. + if (search.Length is null && search.Searched) + { + search = search with + { + EndObjAfterDeclared = declaredEnd >= absoluteDataStart && declaredEnd <= (search.NextObject ?? _source.Length) && + EndObjFollows(absoluteDataStart, declaredEnd, buffered), + }; } + _endStreamSearches[(absoluteDataStart, deferred)] = (search, declaredEnd); + return search; + } + + /// + /// Searches, once for each stream, the data that starts at for its first + /// endstream, and reads what its object's endobj does when none is found; what follows the declared length + /// is left to . + /// + private StreamEndSearch SearchEndStreamOnce(int number, long absoluteDataStart, ReadOnlySpan buffered, bool deferred) + { var nextObject = NextObjectStart(number, absoluteDataStart); // Once the searches have read as much of the file as they may, the bytes the parser holds are searched alone: they @@ -1810,7 +1837,7 @@ StreamEndSearch IPdfStreamDataProvider.FindEndStream( // Past int.MaxValue bytes, no length the reader holds a stream by reaches the endstream found: only a file of more // than 2 GB can put one there, and the stream is read as one whose endstream is not found. With none, what the // object's endobj does is read once with the search, as long as the searches may read. - var search = dataEnd >= 0 && dataEnd - absoluteDataStart <= int.MaxValue + return dataEnd >= 0 && dataEnd - absoluteDataStart <= int.MaxValue ? new StreamEndSearch( (int)(dataEnd - absoluteDataStart), stop, @@ -1818,13 +1845,8 @@ StreamEndSearch IPdfStreamDataProvider.FindEndStream( : new StreamEndSearch(null, stop, EndObjState.Unknown, searched) { EndObjSeen = endObjSeen, - EndObjAfterDeclared = searched && declaredEnd >= absoluteDataStart && declaredEnd <= end && - EndObjFollows(absoluteDataStart, declaredEnd, buffered), ObjectEnd = searched ? EndObjBefore(absoluteDataStart, end, buffered) : -1, }; - - _endStreamSearches[(absoluteDataStart, deferred)] = search; - return search; } /// @@ -2059,6 +2081,8 @@ internal void Close() _streamLengthFaults.TrimExcess(); _endStreamSearches.Clear(); _endStreamSearches.TrimExcess(); + _endStreamChecks.Clear(); + _endStreamChecks.TrimExcess(); Release(_lengthFaultsReported); Release(_syntaxReported); Release(_limitsReached); @@ -3557,8 +3581,8 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // Data a search found no endstream for, before the next object the sections read so far place or the end of the // file, ends nowhere the section says: its rows are not read (#220). - if (deferred is { } unended && _endStreamSearches.TryGetValue((xrefStream.Data.Position, true), out var search) && - search.Length is null) + if (deferred is { } unended && _endStreamSearches.TryGetValue((xrefStream.Data.Position, true), out var known) && + known.Search is { Length: null } search) { var where = search.NextObject is { } next ? string.Create(CultureInfo.InvariantCulture, $"before the next object, at {next}") diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 52de4911..e851a8be 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -699,8 +699,9 @@ private PdfStream ReadStream(PdfDictionary dictionary) _endObjUnknown = false; // The end-of-line after "stream" is not data. A buffer that ends on the keyword, or on the carriage - // return of a CR LF, cannot say where the data starts; a larger one can. - if (dataStart >= span.Length && (dataStart == keywordEnd || span[dataStart - 1] != (byte)'\n')) + // return of a CR LF, cannot say where the data starts; a larger one can — unless the buffer ends the data, and the + // stream's data, which starts there, holds nothing. + if (!_endsData && dataStart >= span.Length && (dataStart == keywordEnd || span[dataStart - 1] != (byte)'\n')) { _truncated = true; } @@ -875,6 +876,13 @@ private PdfStream FinishWithoutEndStream( if (end.Kind != StreamEndKind.Undecided) { + // What follows an endstream the buffer holds, when the buffer ends before it shows, is read on in the file, + // as it is past the buffer: where the window ends decides nothing of the object's endobj (#174). + if (end is { Kind: StreamEndKind.EndStream, EndObj: EndObjState.Unseen } && _streamData is not null) + { + _endObjAfterStream = _streamData.CheckEndStream(_baseOffset + dataEnd); + } + return end.Kind == StreamEndKind.EndStream; } } diff --git a/src/AdCodicem.Pdf/IO/StreamLengthFault.cs b/src/AdCodicem.Pdf/IO/StreamLengthFault.cs index 1b3f110e..72ccb482 100644 --- a/src/AdCodicem.Pdf/IO/StreamLengthFault.cs +++ b/src/AdCodicem.Pdf/IO/StreamLengthFault.cs @@ -50,8 +50,9 @@ internal readonly record struct StreamLengthFault /// /// Gets the offset of the object the search for the endstream stopped at — where an index places it, or where - /// its header starts in the file's bytes —, when is - /// ; null otherwise. + /// its header starts in the file's bytes —, when it stopped at one: is then + /// , or when + /// the object's own endobj ended the data before it; null otherwise. /// public long? NextObject { get; init; } diff --git a/tests/AdCodicem.Pdf.Tests/ReadOrderTests.cs b/tests/AdCodicem.Pdf.Tests/ReadOrderTests.cs index 387015c2..a1147d4c 100644 --- a/tests/AdCodicem.Pdf.Tests/ReadOrderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ReadOrderTests.cs @@ -1911,4 +1911,43 @@ private static List Findings(PdfDocument document, int validations) return first; } } + + [Fact] + public void A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does() + { + // Object 4 lost its endstream, its endobj after its 10 bytes; its /Length, object 5, gives 10 where the table places + // it, and 3 where a second definition, which only a rebuild finds, gives it. Read before the rebuild, then after it, + // object 4 was served 3 bytes, taken as the length its endobj follows — what was read of the endobj after the first + // length was kept with the search. Read again, it ends as it does when first read after the rebuild: where its + // object's endobj ends it, after 10 bytes (#174). + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + }; + rows[4] = file.Position; + file.Raw("4 0 obj\n<< /Length 5 0 R >>\nstream\nABCDEFGHIJ\nendobj\n"); + rows[5] = file.Object(5, "10"); + rows[7] = file.Padding(7, 2000); + file.Object(6, "(six)"); + file.Object(5, "3"); + rows[6] = rows[1] + 1; + file.Table(8, rows, "/Size 8 /Root 1 0 R"); + + using var before = PdfDocument.Open(file.Bytes); + before.GetObject(new PdfObjectId(4)).AsStream().Required().GetRawBytes().ToArray().Should().Equal("ABCDEFGHIJ"u8.ToArray()); + before.GetObject(new PdfObjectId(6)); + before.WasRepaired.Should().BeTrue(); + var again = before.GetObject(new PdfObjectId(4)).AsStream().Required(); + + using var after = PdfDocument.Open(file.Bytes); + after.GetObject(new PdfObjectId(6)); + var first = after.GetObject(new PdfObjectId(4)).AsStream().Required(); + + first.GetRawBytes().ToArray().Should().Equal("ABCDEFGHIJ"u8.ToArray()); + again.GetRawBytes().ToArray().Should().Equal("ABCDEFGHIJ"u8.ToArray()); + before.Reader.TryGetStreamLengthFault(4, out var fault).Should().BeTrue(); + fault.Taken.Should().Be(10); + } } diff --git a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs index 36e01bfc..5f521595 100644 --- a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs +++ b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs @@ -2068,4 +2068,25 @@ public override int Read(long offset, Span buffer) return count; } } + + [Theory] + [InlineData("stream")] + [InlineData("stream\r")] + [InlineData("stream\n")] + [InlineData("stream\r\n")] + public void A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so(string tail) + { + // A buffer that ends on the keyword, or on the carriage return of a CR LF, cannot say where the data starts, and was + // left for a larger one, which a buffer that ends the file has none of: the stream read as no data, in silence. + // Its data starts where the file ends, holding nothing, as with the line feed after the keyword (#174). + var text = "%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R /Metadata 5 0 R >> endobj\n" + + "2 0 obj << /Type /Pages /Kids [] /Count 0 >> endobj\n5 0 obj << /Length 10 >>\n" + tail; + using var document = PdfDocument.Open(Encoding.ASCII.GetBytes(text)); + + document.GetObject(new PdfObjectId(5)).AsStream().Required().RawLength.Should().Be(0); + document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.StreamTruncated).Which.Message.Should().Be( + "The stream declared 10 bytes, and no endstream follows its data before the end of the file; the 0 bytes to the end of the file are taken as its data."); + document.Reader.TryGetStreamLengthFault(5, out var fault).Should().BeTrue(); + fault.EndStream.Should().Be(EndStreamState.MissingBeforeEndOfFile); + } } diff --git a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs index b327b1b8..4bfba319 100644 --- a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs +++ b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs @@ -341,6 +341,38 @@ public void An_object_that_lost_its_end_ends_at_the_next_header_wherever_the_win failures.Should().BeEmpty(); } + [Theory] + [InlineData(-40)] + [InlineData(-20)] + [InlineData(-3)] + [InlineData(0)] + [InlineData(5)] + public void What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls(int shift) + { + // An integer follows object 4's endstream in place of its endobj, starting shift bytes past the window's edge: where + // the edge cut that integer, or the white space before it, the object's endobj was judged unseen, and its loss went + // unreported. What follows the keyword is read on in the file, as past the buffer (#174). + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + }; + var start = file.Position; + rows[4] = start; + const string Before = "4 0 obj\n<< /Length 0000 >>\nstream\n"; + const string After = "\nendstream "; + var length = Window + shift - Before.Length - After.Length; + file.Raw(Before.Replace("0000", length.ToString("D4", CultureInfo.InvariantCulture), StringComparison.Ordinal) + + new string('a', length) + After + "7777777777\n"); + rows[6] = file.Object(6, "(six)"); + file.Table(7, rows, "/Size 7 /Root 1 0 R"); + using var document = PdfDocument.Open(file.Bytes); + + document.GetObject(new PdfObjectId(4)).AsStream().Required().RawLength.Should().Be(length); + document.Reader.IsEndObjMissing(4, out _).Should().BeTrue(); + } + [Fact] public void An_object_whose_header_lies_past_the_window_is_read_where_the_index_says() { From 3f76c835ea5d01ade320f7c2d2ce7bc6255e5309 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 07:53:40 +0000 Subject: [PATCH 013/146] fix: name the object a guard cut at the end of a /Length's chain, and keep no length from it A /Length that leads through a reference to an object a guard cut is read as the guard's: what the cut left is only a place to look for endstream at. The review of the batch found two places where the length's fault still said otherwise. - The fault named the object the /Length refers to, not the one the guard cut at the end of the chain: "names object 20 0, which one of the reader's limits cut" of an object read whole. - The integer the cut left, a parted 1 of 15, was kept as the fault's declared length. It is now only where the parser looks for endstream first, and the fault declares no length. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 20 +++++++++++--- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 27 +++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index e851a8be..10037027 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -715,7 +715,7 @@ private PdfStream ReadStream(PdfDictionary dictionary) } // An integer read from a /Length a guard cut is a place to check, never a length kept when nothing confirms it. - var checkedAt = declared.Form == StreamLengthForm.ObjectCut ? (int)declared.Value.GetValueOrDefault(-1) : length; + var checkedAt = declared.Hint ?? length; var confirmed = ConfirmsLength(span, dataStart, checkedAt); if (confirmed is null) @@ -976,11 +976,14 @@ private readonly DeclaredLength ReadLength(PdfDictionary dictionary) presence = value is PdfNull ? ObjectPresence.Unproduced : ObjectPresence.Defined; } - // An integer the cut left, whole or parted, is only a place to look for endstream at (#224). + // An integer the cut left, whole or parted, is only a place to look for endstream at, and no declared length; the + // object the guard cut is the one a chain of references from the /Length leads to (#224). if (presence == ObjectPresence.Cut) { return new DeclaredLength( - StreamLengthForm.ObjectCut, value.AsInteger() is { } hint and >= 0 and <= int.MaxValue ? hint : null, reference); + StreamLengthForm.ObjectCut, + Reference: named.Target(), + Hint: value.AsInteger() is { } hint and >= 0 and <= int.MaxValue ? (int)hint : null); } if (presence != ObjectPresence.Defined) @@ -1529,6 +1532,15 @@ public void Report(string code, string message, long position) => /// The object it names, when it is a reference. /// What kind of value it holds, when that is not an integer. /// The real number, name or boolean it holds, as the file wrote it; null for any other value. + /// + /// The integer an object a guard cut held before the cut, when it is one a stream can have: where to look for an + /// endstream first, and no declared length (#224). + /// private readonly record struct DeclaredLength( - StreamLengthForm Form, long? Value = null, PdfObjectId? Reference = null, string? Kind = null, PdfObject? Written = null); + StreamLengthForm Form, + long? Value = null, + PdfObjectId? Reference = null, + string? Kind = null, + PdfObject? Written = null, + int? Hint = null); } diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 84a5a323..b7968c19 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -375,4 +375,31 @@ private static long FlateObjectStream( CultureInfo.InvariantCulture, $"<< /Type /ObjStm /N {members.Length}{first} {entries}/Filter {filter} /Length {data.Length} >>\nstream\n{Encoding.Latin1.GetString(data)}\nendstream")); } + + [Fact] + public void A_missing_endstream_names_the_object_a_guard_cut_at_the_end_of_the_length_s_chain_and_keeps_no_length() + { + // Object 4's /Length, object 20, holds a reference to object 21, which MaxObjectLength cuts inside "15": the report + // of its missing endstream named object 20, read whole, and the fault kept the parted 1 as the declared length. + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, Page), + [4] = file.Object(4, "<< /Length 20 0 R >>\nstream\n0 0 m 10 10 l S"), + [20] = file.Object(20, "21 0 R"), + [21] = file.Object(21, new string(' ', 1024 - "21 0 obj\n".Length - 1) + "15" + new string(' ', 100)), + }; + file.Table(22, rows, "/Size 22 /Root 1 0 R"); + using var document = PdfDocument.Open(file.Bytes, ObjectGuard); + + document.GetObject(new PdfObjectId(4)).AsStream().Required().GetRawBytes().ToArray().Should().Equal("0 0 m 10 10 l S"u8.ToArray()); + document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.StreamTruncated) + .Which.Message.Should().StartWith("The stream's /Length names object 21 0, which one of the reader's limits cut"); + document.Reader.TryGetStreamLengthFault(4, out var fault).Should().BeTrue(); + fault.Form.Should().Be(StreamLengthForm.ObjectCut); + fault.Reference.Should().Be(new PdfObjectId(21)); + fault.Declared.Should().BeNull(); + } } From 7bed3a2c58c6521fb02dc470168d1468e95a83ec Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 08:25:23 +0000 Subject: [PATCH 014/146] fix: end the chain for the guard at a /Prev or /XRefStm that leads to an object it cut A cross-reference stream's /W, /Index, /Size, /Filter or /DecodeParms that leads to an object MaxObjectLength cuts became the guard's in this batch. A /Prev or /XRefStm written as a reference to such an object was left out. The section it names was reported missing as the file's fault, the index was rebuilt, and the validator said no section placed the object, though the newer section did. Now the chain ends there for the guard, as for the other values. Nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read: reported, or thrown at opening under ThrowOnLimit. A hybrid file's stream so hidden ends the chain with it, since the older sections' rows would stand for its own. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 33 ++++++++++++++++--- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 16 +++++---- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 22 +++++++++++++ 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 2ac0f86d..630c2544 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3051,8 +3051,12 @@ private bool TryReadClassicTable(long absolute, XRefSectionRecord section, out l if (ReadTrailer(window.Memory, absolute, lexer.Position, windowFull, section) is { } trailer) { MergeSectionTrailer(trailer, section); - previous = SectionOffset(trailer, PdfName.Prev, absolute); - hybrid = SectionOffset(trailer, PdfName.XRefStm, absolute); + previous = SectionOffset(trailer, PdfName.Prev, absolute, out _); + hybrid = SectionOffset(trailer, PdfName.XRefStm, absolute, out var streamHidden); + + // A stream a guard hid holds rows newer than the older sections': the chain ends with it, as with a + // stream a guard cut, since their rows would stand for the stream's. + previous = streamHidden ? -1 : previous; } return true; @@ -3785,7 +3789,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect _rowsLeftUnread = RowsFrom(index is null ? [0, declared] : subsections, stopped, rowsCut ? long.MaxValue : held); _xref.MergeTrailer(dictionary); - previous = rowsCut ? -1 : SectionOffset(dictionary, PdfName.Prev, absolute); + previous = rowsCut ? -1 : SectionOffset(dictionary, PdfName.Prev, absolute, out _); return XRefSectionState.Read; } @@ -4070,10 +4074,18 @@ public bool TryRead(PdfName key, PdfObject? written, bool entry, out PdfObject? /// /Prev 576066 0 R, naming an object the file lacks — names no section, and the sections it should have named /// are reported missing, to be looked for by rebuilding the index. A negative integer is no offset either. Once a guard /// filled the index, though, a reference that leads to what the guard hides () names what is - /// unknown, and the chain ends there for the guard, which is reported, not for the file (#175). + /// unknown, and the chain ends there for the guard, which is reported, not for the file (#175). So does one that leads + /// to an object cut, as a cross-reference stream's values do: the guard is + /// reached once the chain is read (#263). /// - private long SectionOffset(PdfDictionary trailer, PdfName key, long section) + /// The section's trailer, or its stream's dictionary. + /// The entry to read, /Prev or /XRefStm. + /// The offset of the section, which a report names. + /// Whether the entry leads to an object a guard cut, which ends the chain. + private long SectionOffset(PdfDictionary trailer, PdfName key, long section, out bool hidden) { + hidden = false; + var value = trailer.GetRaw(key); if (value is null) @@ -4095,6 +4107,17 @@ private long SectionOffset(PdfDictionary trailer, PdfName key, long section) return -1; } + // So does one leading to an object a guard cut, whose loading, once the chain is read, reaches the guard (#263). The + // older sections, and their trailers, are the guard's to hide. + if (value is PdfReference && Follow(key, value, entry: true, out _, out var cut) is not null && cut is { } past) + { + _cutByGuard = true; + _structure.TrailersHiddenByGuard = true; + (_chainValuesCut ??= []).Add(past); + hidden = true; + return -1; + } + _indexIncomplete = true; _structure.Add(new XRefSectionRecord(FileQuote.Name(key), -1, section) { diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 38e4e912..bd035420 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -315,9 +315,10 @@ public static byte[] UpdatesWithIndirectLengths(int count, int padding) /// /// A hybrid file: a classic table indexes the catalog, the page tree and object 6, which holds the offset of /// cross-reference stream 7; its trailer's /XRefStm is , in which {offset} - /// stands for that offset. Only the stream indexes the page, object 3. + /// stands for that offset. Only the stream indexes the page, object 3. Object 6's value comes after + /// spaces. /// - public static byte[] HybridNamingItsStream(string xrefStm) + public static byte[] HybridNamingItsStream(string xrefStm, int padding = 0) { using var file = new Writer("1.5"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); @@ -326,8 +327,8 @@ public static byte[] HybridNamingItsStream(string xrefStm) // The stream's offset is written in object 6 with a fixed width, so that it can be known before the stream is. var six = file.Position; - var streamAt = six + "6 0 obj\n0000000000\nendobj\n".Length; - file.Object(6, streamAt.ToString("D10", CultureInfo.InvariantCulture)); + var streamAt = six + padding + "6 0 obj\n0000000000\nendobj\n".Length; + file.Object(6, new string(' ', padding) + streamAt.ToString("D10", CultureInfo.InvariantCulture)); var stream = file.Position; var data = Encode([(1, file.OffsetOf(3), 0)], Rows.Raw); file.Stream(7, string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size 7 /W [1 4 2] /Index [3 1] /Length {data.Length}"), data); @@ -342,9 +343,10 @@ public static byte[] HybridNamingItsStream(string xrefStm) /// /// A file of two classic tables: the first indexes the catalog, the page tree and the page; the second, an update, /// indexes object 4 and object 9, which holds the first table's offset, and names that table through - /// , in which {offset} stands for the offset. + /// , in which {offset} stands for the offset. Object 9's value comes after + /// spaces. /// - public static byte[] UpdateNamingItsPrevious(string prev) + public static byte[] UpdateNamingItsPrevious(string prev, int padding = 0) { using var file = new Writer("1.4"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); @@ -355,7 +357,7 @@ public static byte[] UpdateNamingItsPrevious(string prev) CultureInfo.InvariantCulture, $"xref\n0 4\n0000000000 65535 f\r\n{file.OffsetOf(1):D10} 00000 n\r\n{file.OffsetOf(2):D10} 00000 n\r\n{file.OffsetOf(3):D10} 00000 n\r\ntrailer\n<< /Size 4 /Root 1 0 R >>\nstartxref\n{first}\n%%EOF\n")); file.Object(4, "(an update)"); - file.Object(9, first.ToString(CultureInfo.InvariantCulture)); + file.Object(9, new string(' ', padding) + first.ToString(CultureInfo.InvariantCulture)); var second = file.Position; file.Text(string.Create( CultureInfo.InvariantCulture, diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index b7968c19..f3ef84b8 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -402,4 +402,26 @@ public void A_missing_endstream_names_the_object_a_guard_cut_at_the_end_of_the_l fault.Reference.Should().Be(new PdfObjectId(21)); fault.Declared.Should().BeNull(); } + + [Theory] + [InlineData("Prev")] + [InlineData("XRefStm")] + public void A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s(string key) + { + // The newer section's /Prev, object 9, or its /XRefStm, object 6, is one that section places and MaxObjectLength cuts + // before its value: the section it names was said to be missing, and the index rebuilt. The chain ends there for the + // guard, which is reached once the chain is read — thrown at opening when the caller asks for it (#263). + var file = key == "Prev" + ? ChainFiles.UpdateNamingItsPrevious("9 0 R", padding: 3000) + : ChainFiles.HybridNamingItsStream("6 0 R", padding: 3000); + using var document = PdfDocument.Open(file, ObjectGuard); + + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.Reader.Structure.Sections.Should().ContainSingle(); + new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionNotFound).Should().BeFalse(); + + var throwing = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); + throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); + } } From b7129ee222eacbbdc45877c397124f851e06a998 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 08:25:39 +0000 Subject: [PATCH 015/146] fix: leave a header a window's edge cuts short to the edge, and name no object it ends in a trailer Two faults of the header rule, which ends an object's open containers at the next object's header. - Inside a dictionary or an array, a header the window's edge cut short ("6 0", "6 0 ob") was read for what it seemed: its number and generation as keys that are not names, or the number as the value of the key before it. A larger window drops these reports, but where none follows, at MaxObjectLength or in the rebuild's trailer scan, they were kept as the file's faults. The parse now counts as having met the edge there, as it does for a reference the edge cut. - The report said the header ended "the object" in a trailer, or a value read alone, where no object is open. It now says the header came inside the construct left open. A test case now holds the dictionary's value path, where a header stands for a key's value. Refs #230 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 35 +++++++++++++++++-- tests/AdCodicem.Pdf.Tests/ParserTests.cs | 25 ++++++++++++- tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs | 32 +++++++++++++++++ 3 files changed, 88 insertions(+), 4 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 10037027..1f226293 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -91,6 +91,12 @@ internal ref struct PdfObjectParser /// private int _objectNumber; + /// + /// Whether the value is an indirect object's, read after its header: a report of the header that ended a container left + /// open names the object it ended, which a trailer, or a value read alone, is not. + /// + private bool _inObject; + /// /// The object stream member the buffer holds, when it is an object stream's decoded data rather than bytes of the /// file; null otherwise. @@ -269,6 +275,7 @@ public bool TryReadIndirectObject(out PdfObjectId id, out PdfObject value) var empty = _lexer.Read().IsKeyword("endobj"u8); _lexer.Position = beforeValue; + _inObject = true; value = ParseObject(); if (empty) @@ -424,8 +431,8 @@ private PdfObject ParseIntegerOrReference(PdfToken token, int depth) } // The buffer ended before the look-ahead could tell: "12 0" may be all there is, or what a - // window's edge left of "12 0 R". - if (third.Kind == PdfTokenKind.EndOfInput) + // window's edge left of "12 0 R" — or, inside a container, of the next object's header, as "12 0 ob" may be. + if (third.Kind == PdfTokenKind.EndOfInput || (depth > 0 && HeaderUndecided(second, third))) { _truncated = true; } @@ -1343,7 +1350,10 @@ private void ReportCut( return; } - var data = atEndObj ? "An endobj ended the object" : atHeader ? "An object header ended the object" : End; + // A trailer, or a value read alone, is no object a header can end: the header came inside the construct left open. + var data = atEndObj ? "An endobj ended the object" + : atHeader ? (_inObject ? "An object header ended the object" : "An object header came") + : End; var content = swallowed > 0 ? string.Create(CultureInfo.InvariantCulture, $", which takes the {swallowed:N0} {(swallowed == 1 ? "byte" : "bytes")} from where it opens to that end") : valueMissing ? ", which was never closed, before the value of its last key" : ", which was never closed"; @@ -1392,9 +1402,28 @@ private bool StartsHeader(PdfToken number) var generation = _lexer.Read(); var keyword = generation.Kind == PdfTokenKind.Integer ? _lexer.Read() : generation; _lexer.Position = saved; + + // Tokens a window's edge leaves undecided may be a header a larger window shows: the parse met the edge, and what it + // reads from here — the number and the generation, as keys — is no fault of the file's. + if (HeaderUndecided(generation, keyword)) + { + _truncated = true; + } + return IsHeader(number, generation, keyword); } + /// + /// Determines whether a window's edge leaves undecided whether a number and the two tokens after it are the next + /// object's header: the edge comes before a generation, before a keyword, or inside one that obj starts with. + /// + private readonly bool HeaderUndecided(PdfToken generation, PdfToken keyword) => + !_endsData && + (generation.Kind == PdfTokenKind.EndOfInput || + (generation.Kind == PdfTokenKind.Integer && + (keyword.Kind == PdfTokenKind.EndOfInput || + (keyword.Kind == PdfTokenKind.Keyword && keyword.End >= _memory.Length && "obj"u8.StartsWith(keyword.Text))))); + /// /// Determines whether three tokens are an object header as the bytes hold one, the rule the search for a stream's /// endstream stops at (): N G obj, digits alone for a number from 1 to diff --git a/tests/AdCodicem.Pdf.Tests/ParserTests.cs b/tests/AdCodicem.Pdf.Tests/ParserTests.cs index abc96759..0355e4d6 100644 --- a/tests/AdCodicem.Pdf.Tests/ParserTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ParserTests.cs @@ -176,7 +176,29 @@ public void Ends_the_containers_an_object_left_open_at_the_next_object_s_header_ id.Should().Be(new PdfObjectId(6)); value.Should().BeOfType(); - diagnostics.Should().ContainSingle().Which.Message.Should().Be("An object header ended the object inside an array, which was never closed."); + diagnostics.Should().ContainSingle().Which.Message.Should().Be("An object header came inside an array, which was never closed."); + } + + [Theory] + [InlineData("<< /A 1\n6")] + [InlineData("<< /A 1\n6 0")] + [InlineData("<< /A 1\n6 0 ")] + [InlineData("<< /A 1\n6 0 o")] + [InlineData("<< /A 1\n6 0 ob")] + [InlineData("<< /A 1\n6 0 obj")] + [InlineData("<< /A 1 /B\n6 0 obj")] + [InlineData("<< /A [1\n6 0 obj")] + public void Tokens_a_window_s_edge_leaves_undecided_as_a_header_are_no_stray_keys(string text) + { + // Inside a dictionary, the next object's header the window's edge cuts short: its number and generation were reported + // as keys that are not names, and kept so wherever no larger window follows, where a larger one reads a header (#230). + var diagnostics = new PdfDiagnostics(); + var parser = new PdfObjectParser(Encoding.ASCII.GetBytes(text), diagnostics: diagnostics, endsData: false); + + parser.ParseObject().Should().BeOfType(); + + parser.IsTruncated.Should().BeTrue(); + diagnostics.Should().BeEmpty(); } [Theory] @@ -197,6 +219,7 @@ public void Takes_an_object_header_a_window_s_edge_reaches_for_none_until_a_larg [Theory] [InlineData("<< /A [1 << /B 2\n6 0 obj\n", 2)] [InlineData("<< /A 1\n6 0 obj\n", 1)] + [InlineData("<< /A\n6 0 obj\n", 1)] public void A_header_where_a_key_or_a_value_is_expected_ends_the_dictionary(string text, int depthOfCut) { var diagnostics = new PdfDiagnostics(); diff --git a/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs b/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs index 8b903ac0..7375496a 100644 --- a/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs +++ b/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs @@ -269,6 +269,38 @@ public void A_trailer_that_lost_its_closing_delimiter_takes_nothing_of_the_updat document.Trailer.ContainsKey(PdfName.Info).Should().BeFalse(); document.Trailer.ContainsKey(PdfName.Type).Should().BeFalse(); new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.ObjectReferenceMissing).Should().BeFalse(); + + // A trailer is no object: the report said the header ended "the object" at the trailer's <<. + document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.SyntaxTruncatedObject) + .Which.Message.Should().Be("An object header came inside a dictionary, which was never closed."); + } + + [Theory] + [InlineData(" /A 1\n\n6 0 obj")] + [InlineData(" /A 1\n\n6 0 ")] + [InlineData(" /A 1 /A\n\n6 0 obj")] + public void A_header_the_guard_s_cut_leaves_undecided_is_the_guard_s(string tail) + { + // Object 4 lost its >> and its endobj, and the next object's header comes where MaxObjectLength cuts it: the header's + // number and generation were kept as keys that are not names, or as the value of the key before them, where a larger + // limit reads the header (#230). + const int Limit = 128; + var head = "4 0 obj\n<< /Pad ("; + var body = "<< /Pad (" + new string('x', Limit - head.Length - (")" + tail).Length) + ")" + tail + "\n<< /Foo 7 >>"; + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + [4] = file.Object(4, body), + }; + file.Table(5, rows, "/Size 5 /Root 1 0 R"); + using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = Limit } }); + + document.GetObject(new PdfObjectId(4)); + + document.Diagnostics.Should().NotContain(entry => entry.Code == PdfDiagnosticCodes.SyntaxUnexpectedToken); + document.Diagnostics.Should().Contain(entry => entry.Code == PdfDiagnosticCodes.LimitObject); } /// From d3c44d148df6efcf2f6b72b93af4db09d3572bad Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 08:25:52 +0000 Subject: [PATCH 016/146] fix: settle a cross-reference stream's deferred /Length through its references and its guards The review of the batch found four ways a /Length the chain could not read was settled wrongly. - Only the first link of a chain of references was read while the chain was. A /Length object holding "6 0 R", where object 6 waited for the chain or was cut by MaxObjectLength, was judged to hold null, and the guard was never reached. The whole chain is followed now, and the length waits for the chain wherever a link cannot be read. - Taking the data again to the section's own row for the /Length used the first row numbering it, a row of a reserved type included, which reading the rows ignores. It now uses the row the reading keeps. - When MaxObjectLength cut that /Length object, the data was not taken again, and the rows past the endstream they spell were charged to the file: the section was said to lack them, and an older revision's rows stood for them. The integer the cut left is now where an endstream is looked for. When none confirms it, those rows are the guard's, as rows past a decoding guard are, and the chain ends there. - Data MaxStreamLength cut was still checked against the length once the chain was read, the guard's cut reported as the file's. Whether a guard cut it is known only once decoding read it, and it is checked then. Refs #220, #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 140 ++++++++++++------ tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 32 +++- .../DocumentReaderTests.cs | 40 +++++ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 55 +++++++ 4 files changed, 222 insertions(+), 45 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 630c2544..697bb03c 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -751,6 +751,24 @@ public PdfObject GetObject(PdfObjectId id) } } + /// + /// Reads, while the chain is read, what object leads to through as many references as + /// follows, each read with ; null when one of them cannot be read + /// — no section read so far places it, or a guard cut it —: a value whose chain leads there waits for the chain, as one + /// the first reference cannot read does. + /// + private PdfObject? ReadThroughWithoutLoading(PdfObjectId id) + { + var read = ReadWithoutLoading(id); + + for (var link = 1; read is PdfReference next && link < MaxReferenceLinks; link++) + { + read = ReadWithoutLoading(next.Id); + } + + return read; + } + /// /// Says whether the file holds object and whether the reader could produce it, as /// of the last time it was asked for: call it after resolving the object. @@ -1856,9 +1874,9 @@ PdfObject IPdfStreamDataProvider.ResolveLength(PdfReference reference, out Objec if (_readingChain) { - // A /Length no section read so far places is read once the chain is read: the data is found by its endstream - // meanwhile, and the length checked then (#182). - var read = ReadWithoutLoading(id); + // A /Length no section read so far places — or one a chain of references from it leads to, or one a guard cut — is + // read once the chain is read: the data is found by its endstream meanwhile, and the length checked then (#182). + var read = ReadThroughWithoutLoading(id); presence = read is null ? ObjectPresence.Deferred : ObjectPresence.Defined; return read ?? PdfNull.Instance; } @@ -3577,9 +3595,9 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // A /Length no section read so far places: the data was taken up to the first endstream it holds, and the /Length is // read, and checked, once the chain is, if the section is read — or before its rows are, where the section places - // it among them. Data a guard cut is the guard's, which says so (ADR 34), and is not checked against a length. - var deferred = dictionary.GetRaw(PdfName.Length) is PdfReference length && ReadWithoutLoading(length.Id) is null && - !section.CutByLimit && !xrefStream.Data.CutByGuard + // it among them. So is one a chain of references from it leads to, or a guard cut, which the check then passes over. + var deferred = dictionary.GetRaw(PdfName.Length) is PdfReference length && ReadThroughWithoutLoading(length.Id) is null && + !section.CutByLimit ? new DeferredLength(length.Id, xrefStream.Data.Length, number) : (DeferredLength?)null; @@ -3688,6 +3706,9 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect long held; var retaken = false; + // Whether a guard cut the /Length the section's own row places, which leaves how far the data runs unknown. + var lengthCut = false; + while (true) { var decoded = Filters.PdfFilterPipeline.Decode(xrefStream, _diagnostics, out var leftEncoded, out var cut); @@ -3713,14 +3734,17 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // Fewer rows than the section declares, in data a /Length the chain could not read left to the first endstream: // where the section's own row for that /Length, among those held, gives more data, which an endstream confirms, // the rows spelled an endstream, and the data is taken again to that length (#220). - if (!retaken && deferred is { } unsettled && declared > held && decodedWhole && row > 0 && - RetakeDeferred(xrefStream, unsettled, data, fieldWidths, row, held, index is null ? [0, declared] : subsections) is { } whole) + if (!retaken && deferred is { } unsettled && declared > held && decodedWhole && row > 0) { - _diagnostics.RollBack(decoding); - xrefStream = whole; - deferred = unsettled with { Taken = whole.Data.Length }; - retaken = true; - continue; + if (RetakeDeferred(xrefStream, unsettled, data, fieldWidths, row, held, index is null ? [0, declared] : subsections, out lengthCut) is { } whole) + { + _diagnostics.RollBack(decoding); + xrefStream = whole; + deferred = unsettled with { Taken = whole.Data.Length }; + retaken = true; + lengthCut = false; + continue; + } } break; @@ -3755,7 +3779,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect if (declared > held) { - if (decodedWhole) + if (decodedWhole && !lengthCut) { // Rows the data does not hold, when the reader decoded all of it, are the file's to answer for, and the // objects they stood for the index's to find. Every fault met before returned, and a refused row is kept @@ -3768,14 +3792,19 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect } else { - // Rows past what MaxDecodedStreamLength let the data decode to are the guard's: what they index reads as - // null until it is lifted, and the chain ends here, since an older section's rows would stand for them. + // Rows past what MaxDecodedStreamLength let the data decode to are the guard's — or past the endstream the + // rows spelled, when a guard cut the /Length the section's own row places, which would say how far the data + // runs —: what they index reads as null until it is lifted, and the chain ends here, since an older + // section's rows would stand for them. _cutByGuard = true; + section.CutByLimit = true; rowsCut = true; } } - if (deferred is { } check) + // Data a guard cut is the guard's, which says so (ADR 34), and is not checked against a length: decoding it, which + // reads it, says whether it was. + if (deferred is { } check && !xrefStream.Data.CutByGuard) { (_deferredLengths ??= [])[xrefStream.Data.Position] = check; } @@ -3801,14 +3830,25 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect /// endstream that confirms it. Null otherwise: the length is checked once the chain is read. /// /// - /// The object is read where the row places it as reads one: nothing is corrected, - /// cached, recorded or reported, and one a guard cuts is not read. A row that does not stand — a newer section gave the - /// number, or refused it; the row is the second for the number, places it in an object stream, or under another - /// generation — leaves the data as it was taken. + /// The section's row for the object is the first that numbers it with a type the reading of its rows takes: one of a + /// reserved type is ignored there, as the specification says (). The object is read + /// where that row places it as reads one: nothing is corrected, cached, recorded or + /// reported. One a guard cuts gives what the cut left of it, an integer read before the cut only a place to look for + /// endstream at (#224), and says so when no endstream confirms it. A row that + /// does not stand — a newer section gave the number, or refused it; the row lies past those held, places the object in + /// an object stream, or under another generation — leaves the data as it was taken. /// private PdfStream? RetakeDeferred( - PdfStream stream, DeferredLength deferred, ReadOnlySpan data, ReadOnlySpan widths, int row, long held, long[] subsections) + PdfStream stream, + DeferredLength deferred, + ReadOnlySpan data, + ReadOnlySpan widths, + int row, + long held, + long[] subsections, + out bool lengthCut) { + lengthCut = false; var number = deferred.Length.Number; if (_xref.TryGet(number, out _) || _refusedNumbers?.Contains(number) == true) @@ -3816,40 +3856,53 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect return null; } - // The rows before the first that numbers the object, in the order the subsections give them: their counts sum within - // a long, as the section's do. - long before = 0; + // The rows that number the object, in the order the subsections give them, up to the first of a type the reading + // takes: their counts sum within a long, as the section's do. + long ordinal = 0; + ulong type = 0; + ulong offset = 0; + ulong generation = 0; var numbered = false; for (var i = 0; i < subsections.Length && !numbered; i += 2) { - numbered = number >= subsections[i] && number - subsections[i] < subsections[i + 1]; - before += numbered ? number - subsections[i] : subsections[i + 1]; + if (number >= subsections[i] && number - subsections[i] < subsections[i + 1]) + { + var at = ordinal + (number - subsections[i]); + + if (at >= held) + { + return null; + } + + // The row lies within the data, whose length an int holds. + var fields = data.Slice((int)at * row, row); + var cursor = 0; + type = widths[0] == 0 ? 1 : ReadField(fields, ref cursor, widths[0]); + offset = ReadField(fields, ref cursor, widths[1]); + generation = ReadField(fields, ref cursor, widths[2]); + numbered = type <= 2; + } + + ordinal += subsections[i + 1]; } - if (!numbered || before >= held) + if (!numbered || type != 1 || offset > long.MaxValue || generation != (ulong)deferred.Length.Generation) { return null; } - // The row lies within the data, whose length an int holds. - var fields = data.Slice((int)before * row, row); - var cursor = 0; - var type = widths[0] == 0 ? 1 : ReadField(fields, ref cursor, widths[0]); - var offset = ReadField(fields, ref cursor, widths[1]); - var generation = ReadField(fields, ref cursor, widths[2]); + var read = TryReadUnrecorded(deferred.Length, PositionOf((long)offset), out var value, out var pastGuard); var dataStart = stream.Data.Position; - if (type != 1 || offset > long.MaxValue || generation != (ulong)deferred.Length.Generation || - !TryReadUnrecorded(deferred.Length, PositionOf((long)offset), out var value, out _) || - value.AsInteger() is not { } length || length <= deferred.Taken || length > int.MaxValue || - length > _source.Length - dataStart || - ((IPdfStreamDataProvider)this).CheckEndStream(dataStart + length) is null) + if ((read || pastGuard) && value.AsInteger() is { } length && length > deferred.Taken && length <= int.MaxValue && + length <= _source.Length - dataStart && ((IPdfStreamDataProvider)this).CheckEndStream(dataStart + length) is not null) { - return null; + return new PdfStream(stream.Dictionary, ((IPdfStreamDataProvider)this).Create(dataStart, (int)length)); } - return new PdfStream(stream.Dictionary, ((IPdfStreamDataProvider)this).Create(dataStart, (int)length)); + lengthCut = pastGuard; + return null; } /// @@ -4735,7 +4788,8 @@ private bool TryParseNumberedAt( /// Parses object at an exact offset, as does, but keeps nothing of the /// reading: what the parser notices is dropped, nothing is recorded, and an object that still runs past the window /// once allows no larger one is not read rather than reported, which - /// says. A stream is not read: whatever its data, the method returns false. + /// says, then giving what the cut left of it. A stream is not + /// read: whatever its data, the method returns false. /// private bool TryReadUnrecorded(PdfObjectId id, long offset, out PdfObject value, out bool pastGuard) { @@ -4772,7 +4826,9 @@ private bool TryReadUnrecorded(PdfObjectId id, long offset, out PdfObject value, { if (windowSize >= maxWindow) { + // What the cut left of the object, which a caller may take as a place to look at, never as its value. pastGuard = true; + value = read && found == id && parsed is not PdfStream ? parsed : PdfNull.Instance; return false; } diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index bd035420..c823b54c 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -187,15 +187,41 @@ byte[] Data(long lengthAt, long streamAt) } } + /// + /// A file of one cross-reference stream, object 5, whose /Length names object 4, which only that stream indexes, + /// in a row before two rows of reserved types that spell endstream after an end-of-line; the page's row comes + /// after them. With , the /Index numbers object 4 twice: first in a row of a + /// reserved type, which the specification says to ignore, then in the row that places it. The file is valid. + /// + public static byte[] AloneWithIndirectLengthBeforeSpelledRows(bool reservedRowFirst) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + var rowCount = reservedRowFirst ? 9 : 8; + file.Object(4, (rowCount * 7).ToString(CultureInfo.InvariantCulture)); + var stream = file.Position; + var reserved = reservedRowFirst ? [(3, 0L, 0)] : Array.Empty<(int, long, int)>(); + var data = new List(Encode([(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), .. reserved, (1, file.OffsetOf(4), 0), (1, stream, 0)], Rows.Raw)); + data.AddRange("\nendstream\0\0\0\0"u8.ToArray()); + data.AddRange(Encode([(1, file.OffsetOf(3), 0)], Rows.Raw)); + var index = reservedRowFirst ? "0 3 4 1 4 2 6 2 3 1" : "0 3 4 2 6 2 3 1"; + file.Stream(5, $"/Type /XRef /Size 8 /W [1 4 2] /Index [{index}] /Root 1 0 R /Length 4 0 R", [.. data]); + file.StartXRef(stream); + return file.ToArray(); + } + /// /// A file of two revisions. The first writes object 8 as (old) and indexes it in cross-reference stream 7. The /// second writes it again as (new), and indexes it in cross-reference stream 10, whose /Length names /// object 4, which only that stream indexes, and whose rows of objects 5 and 6, of reserved types, spell /// endstream after an end-of-line when . With /// , stream 7's /Length names object 9, which only stream 7 indexes, and a comment - /// sets the update further from it than a search for a section's misplaced offset reaches. The file is valid. + /// sets the update further from it than a search for a section's misplaced offset reaches. Object 4 holds + /// , in which {length} stands for the length. The file is valid. /// - public static byte[] UpdateWithIndirectLength(bool spellingEndStream, bool olderIndirect = false) + public static byte[] UpdateWithIndirectLength(bool spellingEndStream, bool olderIndirect = false, string four = "{length}") { using var file = new Writer("1.5"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); @@ -224,7 +250,7 @@ public static byte[] UpdateWithIndirectLength(bool spellingEndStream, bool older file.Object(8, "(new)"); var spelled = spellingEndStream ? "\nendstream\0\0\0\0"u8.ToArray() : Encode([(0, 0, 0), (0, 0, 0)], Rows.Raw); - file.Object(4, (7 + spelled.Length + 14).ToString(CultureInfo.InvariantCulture)); + file.Object(4, four.Replace("{length}", (7 + spelled.Length + 14).ToString(CultureInfo.InvariantCulture), StringComparison.Ordinal)); var second = file.Position; var data = new List(Encode([(1, file.OffsetOf(4), 0)], Rows.Raw)); data.AddRange(spelled); diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 4dd82879..023b754a 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -1855,6 +1855,46 @@ public void Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_ new PdfValidator().Validate(document).Findings.Should().BeEmpty(); } + [Fact] + public void A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it() + { + // The stream's /Index numbers its /Length, object 4, first in a row of a reserved type, which reading the rows ignores, + // then in the row that places it, before rows that spell an endstream: the data was not taken again by the first, and + // the section was said to lack the rows past them, the page's among them (#220). + using var document = PdfDocument.Open(ChainFiles.AloneWithIndirectLengthBeforeSpelledRows(reservedRowFirst: true)); + + document.Diagnostics.Should().BeEmpty(); + document.Reader.Structure.Sections.Should().ContainSingle().Which.Incomplete.Should().BeFalse(); + document.GetObject(new PdfObjectId(3)).AsDictionary().IsOfType(PdfName.Page).Should().BeTrue(); + new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().BeFalse(); + } + + [Fact] + public void A_deferred_length_is_not_checked_against_data_a_guard_cut() + { + // The stream's data, taken up to the endstream its rows spell, runs past MaxStreamLength: decoding cut it, and the + // length, read once the chain is, was checked against it, the guard's cut reported as the file's (ADR 34). + var file = ChainFiles.AloneWithIndirectLengthBeforeSpelledRows(reservedRowFirst: false); + using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxStreamLength = 30 } }); + using var whole = PdfDocument.Open(file); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitStreamLength); + whole.Diagnostics.Should().BeEmpty(); + } + + [Fact] + public void A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is() + { + // The cross-reference stream's /Length, object 5, which the newer table places, holds a reference to object 6, which + // only the stream indexes: the chain read object 5 alone, and judged the length as holding null (#182). + var file = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] /Length 5 0 R", "6 0 R", compressed: false, predicted: false, placed: true, six: "56"); + using var document = PdfDocument.Open(file); + + document.Diagnostics.Should().BeEmpty(); + document.Reader.Structure.Sections.Should().OnlyContain(section => section.State == IO.XRef.XRefSectionState.Read); + } + [Fact] public void Reports_the_rows_a_deferred_length_shows_the_chain_did_not_read() { diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index f3ef84b8..e7b7d23e 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -403,6 +403,61 @@ public void A_missing_endstream_names_the_object_a_guard_cut_at_the_end_of_the_l fault.Declared.Should().BeNull(); } + [Fact] + public void A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read() + { + // The cross-reference stream's /Length, object 5, holds a reference to object 6, which the newer table places and + // MaxObjectLength cuts: the chain read object 5 alone, judged the length as holding null, and never reached the guard + // (#224). + var file = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] /Length 5 0 R", + "6 0 R", + compressed: false, + predicted: false, + placed: true, + placedToo: new Dictionary { [6] = new string(' ', 3000) + "56" }); + using var document = PdfDocument.Open(file, ObjectGuard); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.WasRepaired.Should().BeFalse(); + document.Reader.Structure.Sections.Should().OnlyContain(section => section.State == IO.XRef.XRefSectionState.Read); + + var throwing = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); + throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); + } + + [Fact] + public void The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to() + { + // The update's stream indexes its /Length, object 4, and object 8 after rows that spell an endstream, and + // MaxObjectLength cuts object 4 after its 35: the section was said to lack the rows past the endstream, and object 8 + // read as the first revision wrote it. The 35 the cut left is where to look for the endstream, which confirms it + // (#224). + var file = ChainFiles.UpdateWithIndirectLength(spellingEndStream: true, four: "{length}" + new string(' ', 3000)); + using var document = PdfDocument.Open(file, ObjectGuard); + + document.GetObject(new PdfObjectId(8)).Should().BeOfType().Which.ToText().Should().Be("new"); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.WasRepaired.Should().BeFalse(); + new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().BeFalse(); + } + + [Fact] + public void Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value() + { + // As above, object 4's value lying past the cut: how far the data runs is unknown, and the rows past the endstream are + // the guard's. Object 8 read as the first revision wrote it, its section said to lack them (#224). + var file = ChainFiles.UpdateWithIndirectLength(spellingEndStream: true, four: new string(' ', 3000) + "{length}"); + using var document = PdfDocument.Open(file, ObjectGuard); + + document.GetObject(new PdfObjectId(8)).Should().BeOfType(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.WasRepaired.Should().BeFalse(); + document.Reader.Structure.Sections[0].CutByLimit.Should().BeTrue(); + document.Reader.Structure.Sections[0].Incomplete.Should().BeFalse(); + new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().BeFalse(); + } + [Theory] [InlineData("Prev")] [InlineData("XRefStm")] From 0c0bc400060ddfde0fb604a9fb94f6f5fb3f3fdb Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 08:26:01 +0000 Subject: [PATCH 017/146] test: open the stream length tests' corpus documents under a raised MaxStreamLength The helper that opens a corpus document as the acceptance tests open it mapped every reader limit a manifest entry can raise but the new MaxStreamLength. A document whose entry raises it would have been read under the default by the stream length tests, and under its own bound by the others. Refs #170 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs index 5f521595..878fe37c 100644 --- a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs +++ b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs @@ -1826,6 +1826,7 @@ private static PdfReaderOptions CorpusOptions(CorpusDocument entry) => entry.Rea Limits = PdfReaderLimits.Default with { MaxDecodedStreamLength = raised.MaxDecodedStreamLength ?? PdfReaderLimits.Default.MaxDecodedStreamLength, + MaxStreamLength = raised.MaxStreamLength ?? PdfReaderLimits.Default.MaxStreamLength, MaxObjectLength = raised.MaxObjectLength ?? PdfReaderLimits.Default.MaxObjectLength, MaxXRefSectionLength = raised.MaxXRefSectionLength ?? PdfReaderLimits.Default.MaxXRefSectionLength, MaxXRefSectionCount = raised.MaxXRefSectionCount ?? PdfReaderLimits.Default.MaxXRefSectionCount, From 9597631ce906055963681c7f48e31f3be4d24540 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 09:25:11 +0000 Subject: [PATCH 018/146] fix: take a comment around a stream's end for white space only where it can be The review of the batch found two places where a comment, which ISO 32000-1 (7.2.3) counts as white space outside a stream's data, was read wrongly at the end of one. Both rules were settled with the maintainer. - A /Length too short that ends on a "%" of the data: the rest of the line read as a comment before the endstream, which confirmed the wrong length, and the rest of the data was lost with no report. The same comment let an endobj after it keep the short length when the stream has no endstream. A comment now counts there only once white space comes before it: a "%" right where the length ends is the data's, the length goes unconfirmed, and the search finds the real end. - A stream with no endstream ends at its object's own endobj when only white space follows it up to the next object. A comment there kept the endobj from ending the data: the short length was kept, or the data took in the endobj. Comments after the endobj are white space now. The endobj is followed forward through the reads of the search for endstream, which reads every byte up to where it stops already, rather than read back from there: a comment can only be told from the start of its line. The bytes before the stop are no longer read a second time. Refs #174 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/EndObjTail.cs | 239 ++++++++++++++++++ src/AdCodicem.Pdf/IO/PdfFileReader.cs | 98 ++----- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 5 +- src/AdCodicem.Pdf/IO/StreamEnds.cs | 79 ++---- tests/AdCodicem.Pdf.Tests/ParserTests.cs | 15 ++ .../AdCodicem.Pdf.Tests/StreamLengthTests.cs | 89 +++++++ tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs | 41 +++ 7 files changed, 434 insertions(+), 132 deletions(-) create mode 100644 src/AdCodicem.Pdf/IO/EndObjTail.cs diff --git a/src/AdCodicem.Pdf/IO/EndObjTail.cs b/src/AdCodicem.Pdf/IO/EndObjTail.cs new file mode 100644 index 00000000..6b7ad748 --- /dev/null +++ b/src/AdCodicem.Pdf/IO/EndObjTail.cs @@ -0,0 +1,239 @@ +namespace AdCodicem.Pdf.IO; + +/// +/// Follows, through the reads of a search for endstream in their order, where the data of a stream that has no +/// endstream ends when its object's own endobj ends it: at the last endobj, a whole keyword, after +/// which only white space and comments run to where the search stops, the end-of-line before it left out (#174). +/// +/// +/// +/// A comment after the keyword is white space, as ISO 32000-1 (7.2.3) makes one outside a stream's data: it runs to the end +/// of its line, and an endobj it holds is no keyword. Each byte is read once, however much the reads repeat of the +/// ones before; the last few bytes read are kept, so that a keyword that runs from one read into the next is found, and +/// what comes before it known. +/// +/// +/// Where the search stops is known only once the read that holds the header stopping it is searched, and that header may +/// start in a read before it, which was followed past it. What lies between is the header's runs of digits and white +/// space, which make no keyword: the place where what follows the last endobj stopped being white space and +/// comments says, alone, whether that run reached the header. +/// +/// +internal struct EndObjTail +{ + /// The most bytes before a keyword that what it ends needs: the byte before it, and an end-of-line of two. + private const int Kept = 8; + + private readonly long _dataStart; + private long _upTo; + private Mode _mode; + private long _keywordEnd; + private long _dataEnd; + private long _brokenAt; + private ulong _recent; + private int _recentCount; + + /// Initializes a new instance of the struct, for data that starts at . + public EndObjTail(long dataStart) + { + _dataStart = dataStart; + _upTo = dataStart; + _keywordEnd = -1; + _dataEnd = -1; + _brokenAt = long.MaxValue; + } + + private enum Mode : byte + { + /// No endobj that may end the data is being followed. + Data, + + /// An endobj was read whose next byte says whether it is a whole keyword. + AfterKeyword, + + /// White space follows the last whole endobj. + Tail, + + /// A comment after the last whole endobj runs on. + Comment, + } + + /// + /// Reads the bytes a read of the search holds, from where the reads before stopped to . + /// + /// The bytes of the read. + /// Where they start in the file. + /// Where the search stops in them, or where they end. + public void Read(ReadOnlySpan bytes, long at, long until) + { + var start = (int)Math.Max(0, _upTo - at); + var end = (int)(Math.Min(until, at + bytes.Length) - at); + + if (start >= end) + { + return; + } + + var i = _mode == Mode.Data ? Straddling(bytes, at, start, end) : start; + + while (i < end) + { + switch (_mode) + { + case Mode.AfterKeyword: + // "endobjx" is no keyword; anything else that follows it may be the tail, and is read as such. + if (PdfCharacters.IsRegular(bytes[i])) + { + _brokenAt = at + i; + _mode = Mode.Data; + } + else + { + _mode = Mode.Tail; + } + + break; + + case Mode.Tail: + var next = bytes[i..end].IndexOfAnyExcept(PdfCharacters.Whitespace); + + if (next < 0) + { + i = end; + } + else if (bytes[i + next] == (byte)'%') + { + i += next + 1; + _mode = Mode.Comment; + } + else + { + i += next; + _brokenAt = at + i; + _mode = Mode.Data; + } + + break; + + case Mode.Comment: + // A comment runs to the end of the line; both line endings occur in the wild. + var lineEnd = bytes[i..end].IndexOfAny((byte)'\r', (byte)'\n'); + i = lineEnd < 0 ? end : i + lineEnd + 1; + _mode = lineEnd < 0 ? Mode.Comment : Mode.Tail; + break; + + default: + var hit = bytes[i..end].IndexOf(StreamEnds.EndObjKeyword); + + if (hit < 0) + { + i = end; + } + else + { + i = Found(bytes, at, start, at + i + hit) ? i + hit + StreamEnds.EndObjKeyword.Length : i + hit + 1; + } + + break; + } + } + + Keep(bytes[start..end]); + _upTo = at + end; + } + + /// + /// Gives where the data ends when the search stops at , at an endobj that only white space + /// and comments separate from it; -1 when none does, or the reads did not reach it. + /// + public readonly long DataEnd(long stop) => + stop <= _upTo && _keywordEnd >= 0 && _keywordEnd <= stop && stop <= _brokenAt ? _dataEnd : -1; + + /// + /// Finds an endobj that starts in the bytes the reads before kept and ends in these, and gives where to go on + /// reading these from. + /// + private int Straddling(ReadOnlySpan bytes, long at, int start, int end) + { + var keyword = StreamEnds.EndObjKeyword; + + // The keyword holds no part of itself again: one way at most of parting it fits. + for (var held = 1; held < keyword.Length && held <= _recentCount; held++) + { + var rest = keyword.Length - held; + + if (start + rest > end || !bytes.Slice(start, rest).SequenceEqual(keyword[held..])) + { + continue; + } + + var matches = true; + + for (var k = 0; k < held && matches; k++) + { + matches = Recent(held - 1 - k) == keyword[k]; + } + + if (matches) + { + return Found(bytes, at, start, at + start - held) ? start + rest : start; + } + } + + return start; + } + + /// + /// Takes the endobj that starts at , unless the byte before it makes it part of a longer + /// token, as the one the data may end at; whether the byte after it does is read next. + /// + /// Whether the keyword starts a token. + private bool Found(ReadOnlySpan bytes, long at, int start, long keyword) + { + if (ByteAt(bytes, at, start, keyword - 1) is var before and >= 0 && PdfCharacters.IsRegular((byte)before)) + { + return false; + } + + var dataEnd = keyword; + + if (ByteAt(bytes, at, start, dataEnd - 1) == '\n') + { + dataEnd--; + } + + if (ByteAt(bytes, at, start, dataEnd - 1) == '\r') + { + dataEnd--; + } + + _keywordEnd = keyword + StreamEnds.EndObjKeyword.Length; + _dataEnd = dataEnd; + _brokenAt = long.MaxValue; + _mode = Mode.AfterKeyword; + return true; + } + + /// + /// Gives the byte at in the file: in these bytes from on, among those + /// kept before it; -1 before the start of the data. + /// + private readonly int ByteAt(ReadOnlySpan bytes, long at, int start, long position) => + position < _dataStart ? -1 + : position >= at + start ? bytes[(int)(position - at)] + : Recent((int)(at + start - 1 - position)); + + /// Gives a byte kept, bytes before the last one read. + private readonly byte Recent(int back) => (byte)(_recent >> (8 * back)); + + /// Keeps the last bytes read, the latest in the lowest byte. + private void Keep(ReadOnlySpan read) + { + foreach (var value in read[Math.Max(0, read.Length - Kept)..]) + { + _recent = (_recent << 8) | value; + } + + _recentCount = (int)Math.Min(Kept, _recentCount + (long)read.Length); + } +} diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 697bb03c..a40c3832 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -1611,8 +1611,8 @@ PdfStreamData IPdfStreamDataProvider.Create(long absoluteOffset, int length) /// /// Determines whether an endobj keyword, whole, follows the declared length of a stream whose data starts at - /// , white space and comments between: in the bytes the parser holds when they say, in the - /// file otherwise. + /// , white space and comments between, a comment only after white space: in the bytes the + /// parser holds when they say, in the file otherwise. /// private bool EndObjFollows(long dataStart, long declaredEnd, ReadOnlySpan buffered) => (declaredEnd - dataStart <= buffered.Length @@ -1620,23 +1620,9 @@ private bool EndObjFollows(long dataStart, long declaredEnd, ReadOnlySpan : null) ?? IsEndObjAt(declaredEnd); /// - /// Gives where the data of a stream that has no endstream ends when its object's own endobj ends it before - /// : in the bytes the parser holds when they reach the stop, in the file otherwise. - /// - private long EndObjBefore(long dataStart, long stop, ReadOnlySpan buffered) - { - if (stop - dataStart > buffered.Length) - { - return FindEndObjBefore(dataStart, stop); - } - - var end = StreamEnds.EndObjBefore(buffered[..(int)(stop - dataStart)]); - return end < 0 ? -1 : dataStart + end; - } - - /// - /// Determines whether an endobj keyword, whole, follows after white space and - /// comments: the end of a stream's object that has no endstream. + /// Determines whether an endobj keyword, whole, follows , where a stream's declared + /// length ends, after white space and comments, a comment only after white space (): + /// the end of a stream's object that has no endstream. /// private bool IsEndObjAt(long absoluteOffset) { @@ -1657,53 +1643,6 @@ private bool IsEndObjAt(long absoluteOffset) return next >= 0 && IsEndObjKeywordAt(next); } - /// - /// Gives where the data of a stream that has no endstream ends when its object's own endobj ends it: the - /// endobj, a whole keyword after , that only white space separates from - /// , the end-of-line before it left out; -1 when none does. - /// - /// Where the data starts in the file. - /// Where the search for the stream's endstream stopped: an object, or the end of the file. - private long FindEndObjBefore(long absoluteDataStart, long stop) - { - // The white space before the stop, read back a few bytes at a time and more as it runs on: the search read it - // already, and this reads it once more at most. - var end = stop; - var size = PdfObjectParser.EndObjLookahead; - - while (end > absoluteDataStart) - { - var from = Math.Max(absoluteDataStart, end - size); - using var window = _source.GetWindow(from, (int)(end - from)); - var span = window.Memory.Span; - _endStreamSearchBytes += span.Length; - - if (span.Length < end - from) - { - return -1; - } - - var last = span.LastIndexOfAnyExcept(PdfCharacters.Whitespace); - - if (last >= 0) - { - end = from + last + 1; - break; - } - - end = from; - size = (int)Math.Min(2L * size, EndStreamSearchChunk); - } - - // The keyword, the byte before it and the end-of-line before that. - var before = Math.Max(absoluteDataStart, end - StreamEnds.EndObjKeyword.Length - 3); - Span bytes = stackalloc byte[StreamEnds.EndObjKeyword.Length + 3]; - var length = (int)(end - before); - var held = _source.Read(before, bytes[..length]); - var dataEnd = held == length ? StreamEnds.EndObjEnding(bytes[..length], atDataStart: before == absoluteDataStart) : -1; - return dataEnd < 0 ? -1 : before + dataEnd; - } - /// /// Gives where the first byte at or after lies that is neither white space nor part of a /// comment, reading the file on as far as they run, or the length of the file when they run to its end; -1 when the @@ -1846,11 +1785,18 @@ private StreamEndSearch SearchEndStreamOnce(int number, long absoluteDataStart, // cost no read. var readsFile = _endStreamSearchBytes < EndStreamSearchPasses * _source.Length; var dataEnd = SearchEndStream( - absoluteDataStart, buffered, nextObject ?? _source.Length, deferred, readsFile, out var header, out var searched, out var endObjSeen); + absoluteDataStart, + buffered, + nextObject ?? _source.Length, + deferred, + readsFile, + out var header, + out var searched, + out var endObjSeen, + out var objectEnd); // A header the bytes hold is looked for only before the next object the index places: found, it is the nearer. var stop = header >= 0 ? header : nextObject; - var end = stop ?? _source.Length; // Past int.MaxValue bytes, no length the reader holds a stream by reaches the endstream found: only a file of more // than 2 GB can put one there, and the stream is read as one whose endstream is not found. With none, what the @@ -1863,7 +1809,7 @@ private StreamEndSearch SearchEndStreamOnce(int number, long absoluteDataStart, : new StreamEndSearch(null, stop, EndObjState.Unknown, searched) { EndObjSeen = endObjSeen, - ObjectEnd = searched ? EndObjBefore(absoluteDataStart, end, buffered) : -1, + ObjectEnd = objectEnd, }; } @@ -1984,6 +1930,10 @@ internal static int EndStreamSearchFirstRead(int buffered) => /// Where the header that stopped the search starts, or -1. /// Whether the search reached where it stops, rather than the end of what it might read. /// Whether the bytes searched, before a header that stopped the search, hold an endobj. + /// + /// Where the data ends, when no endstream is found and the search reached where it stops, at the object's own + /// endobj that only white space and comments separate from there (); -1 otherwise. + /// private long SearchEndStream( long dataStart, ReadOnlySpan buffered, @@ -1992,20 +1942,25 @@ private long SearchEndStream( bool readsFile, out long header, out bool searched, - out bool endObjSeen) + out bool endObjSeen, + out long objectEnd) { Span carried = stackalloc byte[HeaderText.CarriedLength]; Span carriedAt = stackalloc long[HeaderText.CarriedLength]; var text = new HeaderText(carried, carriedAt, dataStart); + var tail = new EndObjTail(dataStart); searched = true; + objectEnd = -1; var inBuffer = (int)Math.Clamp(limit - dataStart, 0, buffered.Length); text.Read(buffered[..inBuffer], dataStart); var found = StreamEnds.SearchStretch(text, endsRange: dataStart + inBuffer >= limit, deferred, out header); endObjSeen = StreamEnds.HoldsEndObj(buffered[..inBuffer], header >= 0 ? (int)(header - dataStart) : inBuffer); + tail.Read(buffered[..inBuffer], dataStart, header >= 0 ? header : dataStart + inBuffer); if (found >= 0 || header >= 0) { + objectEnd = found < 0 ? tail.DataEnd(header) : -1; return found; } @@ -2013,6 +1968,7 @@ private long SearchEndStream( if (dataStart + buffered.Length >= end) { + objectEnd = tail.DataEnd(end); return -1; } @@ -2041,9 +1997,11 @@ private long SearchEndStream( text.Read(span, position); found = StreamEnds.SearchStretch(text, endsRange: position + span.Length >= end, deferred, out header); endObjSeen |= StreamEnds.HoldsEndObj(span, header >= 0 ? (int)(header - position) : span.Length); + tail.Read(span, position, header >= 0 ? header : position + span.Length); if (found >= 0 || header >= 0 || position + span.Length >= end) { + objectEnd = found < 0 ? tail.DataEnd(header >= 0 ? header : end) : -1; return found; } diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 1f226293..923a8152 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -939,14 +939,15 @@ private PdfStream FinishWithoutEndStream( var stop = header < 0 ? span.Length : (int)(header - _baseOffset); var stretch = span[dataStart..stop]; - var objectEnd = StreamEnds.EndObjBefore(stretch); + var tail = new EndObjTail(start); + tail.Read(stretch, start, _baseOffset + stop); return new StreamEndSearch(null, header < 0 ? null : header, EndObjState.Unknown) { EndObjSeen = StreamEnds.HoldsEndObj(stretch, stretch.Length), EndObjAfterDeclared = declaredEnd >= start && declaredEnd - _baseOffset <= stop && StreamEnds.IsEndObjAt(span[..stop], (int)(declaredEnd - _baseOffset), reachesEndOfFile: true) == true, - ObjectEnd = objectEnd < 0 ? -1 : start + objectEnd, + ObjectEnd = tail.DataEnd(_baseOffset + stop), }; } diff --git a/src/AdCodicem.Pdf/IO/StreamEnds.cs b/src/AdCodicem.Pdf/IO/StreamEnds.cs index 81541235..258f82d4 100644 --- a/src/AdCodicem.Pdf/IO/StreamEnds.cs +++ b/src/AdCodicem.Pdf/IO/StreamEnds.cs @@ -15,12 +15,19 @@ internal static class StreamEnds public static ReadOnlySpan EndObjKeyword => "endobj"u8; /// - /// Gives where the first byte at or after lies that is neither white space nor part of a - /// comment, or the length of when they run to its end. ISO 32000-1 (7.2.3) makes a comment - /// outside a string or a stream's data count as white space. + /// Gives where the first byte at or after , where a stream's declared length ends, lies that is + /// neither white space nor part of a comment, or the length of when they run to its end. ISO + /// 32000-1 (7.2.3) makes a comment outside a string or a stream's data count as white space; one counts here only after + /// white space, since a % right where the length ends is the data's, a length too short, and the rest of its line + /// the rest of the data. /// - public static int SkipBlanks(ReadOnlySpan bytes, int index) + public static int SkipGap(ReadOnlySpan bytes, int index) { + if (index < bytes.Length && bytes[index] == (byte)'%') + { + return index; + } + var lexer = new PdfLexer(bytes, index); lexer.SkipWhitespaceAndComments(); return lexer.Position; @@ -28,16 +35,15 @@ public static int SkipBlanks(ReadOnlySpan bytes, int index) /// /// Reads, at the start of , whether endstream follows a stream's data — after white - /// space and comments, as many as there are (#174) — and what follows the keyword: endobj, another token, or - /// nothing the bytes show. + /// space and comments, as many as there are (#174), a comment only after white space () — and what + /// follows the keyword: endobj, another token, or nothing the bytes show. /// /// The bytes after the data. /// Whether the bytes end where the file does, which then follows what they hold. public static StreamEnd ReadEnd(ReadOnlySpan bytes, bool reachesEndOfFile) { - var lexer = new PdfLexer(bytes); - lexer.SkipWhitespaceAndComments(); - var at = lexer.Position; + var at = SkipGap(bytes, 0); + var lexer = new PdfLexer(bytes, at); var rest = bytes[at..]; if (!rest.StartsWith(EndStreamKeyword)) @@ -71,12 +77,13 @@ public static StreamEnd ReadEnd(ReadOnlySpan bytes, bool reachesEndOfFile) } /// - /// Determines whether endobj, a whole keyword, starts at once white space and comments are - /// skipped; null when they, or the keyword, run to the end of , which do not end the file. + /// Determines whether endobj, a whole keyword, starts at , where a stream's declared length + /// ends, once white space and comments are skipped (); null when they, or the keyword, run to the + /// end of , which do not end the file. /// public static bool? IsEndObjAt(ReadOnlySpan bytes, int index, bool reachesEndOfFile) { - var at = SkipBlanks(bytes, index); + var at = SkipGap(bytes, index); var rest = bytes[at..]; if (rest.Length <= EndObjKeyword.Length) @@ -99,54 +106,6 @@ public static StreamEnd ReadEnd(ReadOnlySpan bytes, bool reachesEndOfFile) public static bool HoldsEndObj(ReadOnlySpan bytes, int before) => bytes[..Math.Clamp(before, 0, bytes.Length)].IndexOf(EndObjKeyword) >= 0; - /// - /// Gives where the data of a stream that has no endstream ends when its object's own endobj ends it: the - /// endobj, a whole keyword, that only white space separates from the end of , the - /// end-of-line before it left out; -1 when none does. - /// - /// The bytes from the start of the data to where the search for its endstream stopped. - public static int EndObjBefore(ReadOnlySpan stretch) - { - var end = stretch.LastIndexOfAnyExcept(PdfCharacters.Whitespace) + 1; - return EndObjEnding(stretch[..end], atDataStart: true); - } - - /// - /// Gives where the data ends when end with a whole endobj: where the keyword starts, the - /// end-of-line before it left out; -1 when they do not. - /// - /// Bytes that end where the white space before the stop starts. - /// Whether the bytes start where the data does, before which no byte belongs to it. - public static int EndObjEnding(ReadOnlySpan bytes, bool atDataStart) - { - if (!bytes.EndsWith(EndObjKeyword)) - { - return -1; - } - - var keyword = bytes.Length - EndObjKeyword.Length; - - // The keyword is whole: what precedes it ends a token, or is the start of the data. - if (keyword > 0 ? PdfCharacters.IsRegular(bytes[keyword - 1]) : !atDataStart) - { - return -1; - } - - var end = keyword; - - if (end > 0 && bytes[end - 1] == (byte)'\n') - { - end--; - } - - if (end > 0 && bytes[end - 1] == (byte)'\r') - { - end--; - } - - return end; - } - /// /// Searches one read of a search for endstream — the parser's bytes, or a window of the file — for the first /// endstream, and for the first object header before it: returns where the data ends in the file, the diff --git a/tests/AdCodicem.Pdf.Tests/ParserTests.cs b/tests/AdCodicem.Pdf.Tests/ParserTests.cs index 0355e4d6..de588d43 100644 --- a/tests/AdCodicem.Pdf.Tests/ParserTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ParserTests.cs @@ -179,6 +179,21 @@ public void Ends_the_containers_an_object_left_open_at_the_next_object_s_header_ diagnostics.Should().ContainSingle().Which.Message.Should().Be("An object header came inside an array, which was never closed."); } + [Theory] + [InlineData("hello\nendobj\n% a note\n6 0 obj\n", 5)] + [InlineData("hello\nendobj % note endobj\n6 0 obj\n", 5)] + [InlineData("hello\nendobj\n% a note\nx\n6 0 obj\n", -1)] + public void Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow(string data, int length) + { + // Parsed alone, a stream with no endstream ends where its object's endobj ends it, a comment after which is white + // space; a token after it leaves the data running to the next object's header (#174). + var parser = new PdfObjectParser(Encoding.ASCII.GetBytes("4 0 obj\n<< /Length 200 >>\nstream\n" + data)); + + parser.TryReadIndirectObject(out _, out var value).Should().BeTrue(); + + value.AsStream().Required().RawLength.Should().Be(length < 0 ? data.IndexOf("6 0 obj", StringComparison.Ordinal) : length); + } + [Theory] [InlineData("<< /A 1\n6")] [InlineData("<< /A 1\n6 0")] diff --git a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs index 878fe37c..a239679b 100644 --- a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs +++ b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs @@ -2090,4 +2090,93 @@ public void A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_ document.Reader.TryGetStreamLengthFault(5, out var fault).Should().BeTrue(); fault.EndStream.Should().Be(EndStreamState.MissingBeforeEndOfFile); } + + [Fact] + public void A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length() + { + // A /Length 7 bytes short ends on the "%" of the data: the rest of the line read as a comment before the endstream, + // which confirmed the length, and the 7 bytes were lost in silence. A comment counts only after white space. + using var document = PdfDocument.Open(OneStream("<< /Length 3 >>\nstream\nabc%\u00e9\u00ff\u0001xyz\nendstream")); + + document.GetObject(new PdfObjectId(4)).AsStream().Required().RawLength.Should().Be(10); + document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.StreamLengthInvalid) + .Which.Message.Should().Be("The stream declared 3 bytes but ended after 10."); + } + + [Theory] + [InlineData("abc\n% a note\nendstream")] + [InlineData("abc %a note\r\nendstream")] + public void A_comment_after_white_space_where_the_length_ends_confirms_it(string data) + { + // A comment is white space between the data and its endstream (ISO 32000-1, 7.2.3), once white space comes before it. + using var document = PdfDocument.Open(OneStream("<< /Length 3 >>\nstream\n" + data)); + + document.GetObject(new PdfObjectId(4)).AsStream().Required().RawLength.Should().Be(3); + document.Diagnostics.Should().BeEmpty(); + } + + [Fact] + public void A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too() + { + // With no endstream, the endobj after the declared length kept "abc", the comment it skipped lost "%xyz": the endobj + // after the data ends it instead. + using var document = PdfDocument.Open(OneStream("<< /Length 3 >>\nstream\nabc%xyz")); + + document.GetObject(new PdfObjectId(4)).AsStream().Required().RawLength.Should().Be(7); + document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.StreamTruncated) + .Which.Message.Should().EndWith("its object's endobj ends it, after 7 bytes."); + } + + [Theory] + [InlineData("\nendobj\n% a note\n")] + [InlineData("\nendobj % a note\n")] + [InlineData("\nendobj%a note\r")] + [InlineData("\r\nendobj\n%a\n % b endobj x\n\n")] + [InlineData("\nendobj\n% a note")] + public void An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it(string tail) + { + // A comment between the object's endobj and the next object kept the endobj from ending the data: the declared + // length was kept, or the data took in the endobj. A comment after it is white space (ISO 32000-1, 7.2.3). + foreach (var length in new[] { 3, 20_000 }) + { + using var document = PdfDocument.Open(OneStream( + "<< /Length " + length.ToString(CultureInfo.InvariantCulture) + " >>\nstream\nhello" + tail, endObj: false)); + + document.GetObject(new PdfObjectId(4)).AsStream().Required().RawLength.Should().Be(5); + document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.StreamTruncated) + .Which.Message.Should().EndWith("its object's endobj ends it, after 5 bytes."); + document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); + } + } + + [Theory] + [InlineData("\nendobj\n% a note\nfoo\n")] + [InlineData("\nendobj % a note endobj\nx")] + public void An_endobj_a_token_follows_past_its_comments_ends_no_data(string tail) + { + // What follows the endobj past its comments is a token: the endobj is the data's, and the data runs to the next object. + using var document = PdfDocument.Open(OneStream("<< /Length 20000 >>\nstream\nhello" + tail, endObj: false)); + + document.GetObject(new PdfObjectId(4)).AsStream().Required().RawLength.Should().BeGreaterThan(5); + document.Diagnostics.Should().NotContain(entry => entry.Message.EndsWith("after 5 bytes.", StringComparison.Ordinal)); + } + + /// + /// A document whose object 4 is , followed by its endobj when , and + /// then by object 6. + /// + private static byte[] OneStream(string body, bool endObj = true) + { + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + }; + rows[4] = file.Position; + file.Raw("4 0 obj\n" + body + (endObj ? "\nendobj\n" : string.Empty)); + rows[6] = file.Object(6, "(six)"); + file.Table(7, rows, "/Size 7 /Root 1 0 R"); + return file.Bytes; + } } diff --git a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs index 4bfba319..a20a224f 100644 --- a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs +++ b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs @@ -373,6 +373,47 @@ public void What_follows_an_endstream_the_window_holds_is_read_wherever_the_wind document.Reader.IsEndObjMissing(4, out _).Should().BeTrue(); } + [Theory] + [InlineData(-30)] + [InlineData(-21)] + [InlineData(-18)] + [InlineData(-15)] + [InlineData(-13)] + [InlineData(-11)] + [InlineData(-9)] + [InlineData(-7)] + [InlineData(-5)] + [InlineData(-3)] + [InlineData(-1)] + [InlineData(0)] + [InlineData(2)] + [InlineData(4)] + [InlineData(8)] + [InlineData(12)] + public void An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it(int shift) + { + // Object 4 lost its endstream. Its endobj, a comment, and a header no index places, which stops the search, fall + // across the edge of the window and of the first read of the file past it: the keyword parted between two reads, the + // header started in the read before the one that shows it whole. The data ends at the endobj wherever they fall (#174). + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + }; + rows[4] = file.Position; + const string Before = "4 0 obj\n<< /Length 3 >>\nstream\n"; + var length = Window + shift - Before.Length; + file.Raw(Before + new string('a', length) + "\nendobj\n% c\n7 0 obj\n(stray)\nendobj\n"); + rows[6] = file.Object(6, "(six)"); + file.Table(7, rows, "/Size 7 /Root 1 0 R"); + using var document = PdfDocument.Open(file.Bytes); + + document.GetObject(new PdfObjectId(4)).AsStream().Required().RawLength.Should().Be(length); + document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.StreamTruncated) + .Which.Message.Should().EndWith(string.Create(CultureInfo.InvariantCulture, $"its object's endobj ends it, after {length} bytes.")); + } + [Fact] public void An_object_whose_header_lies_past_the_window_is_read_where_the_index_says() { From f31a06df516a26e79eca8533d45163ebdc659e81 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 09:25:21 +0000 Subject: [PATCH 019/146] fix: say a stream is an image only when decoding reaches its image filter past every guard HasImageFilter says whether Decode hands back an encoded image, and read none of the filters' parameters to tell it. Since a value of a step's parameters that leads to an object a guard cut ends the chain where it is read, a Flate step before a DCT one whose /DecodeParms a guard cut left its data Flate-encoded: HasImageFilter still said it was the image, and an extractor that trusts it would have written Flate data as a JPEG. It now reads what decoding reads before the image filter, as the maintainer settled: the parameters of a Flate or LZW step that comes before it, and nothing for a chain that ends sooner. A parameter held in an object stream decodes that stream first, as decoding the stream would. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/Filters/FilterChain.cs | 20 +++++------ .../Objects/PdfStreamDecodingExtensions.cs | 6 ++-- tests/AdCodicem.Pdf.Tests/FilterTests.cs | 26 +++++++------- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 36 +++++++++++++++++++ 4 files changed, 64 insertions(+), 24 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/Filters/FilterChain.cs b/src/AdCodicem.Pdf/IO/Filters/FilterChain.cs index df6b6078..a1dd0f31 100644 --- a/src/AdCodicem.Pdf/IO/Filters/FilterChain.cs +++ b/src/AdCodicem.Pdf/IO/Filters/FilterChain.cs @@ -114,18 +114,19 @@ private FilterChain(in Reading reading) /// The most filters the stream's data may pass through: the chain ends at the element past them. public static FilterChain Resolve(PdfDictionary dictionary, int maxFilters) { - var reading = Read(dictionary, maxFilters, readParameters: true, keep: true); + var reading = Read(dictionary, maxFilters, keep: true); return reading.Filters is null && reading.End == FilterChainEnd.None ? NoFilter : new FilterChain(reading); } /// - /// Resolves what ends the chain of filters the stream describes, reading none of their - /// parameters and keeping nothing. + /// Resolves what ends the chain of filters the stream describes, as decoding resolves it, + /// and keeps nothing: the parameters decoding reads are read too, since a value among them that leads to an object a + /// guard cut, or left out of the index, ends the chain where it is read (#263). /// /// The stream's dictionary. /// The most filters the stream's data may pass through: the chain ends at the element past them. public static FilterChainEnd EndOf(PdfDictionary dictionary, int maxFilters) => - Read(dictionary, maxFilters, readParameters: false, keep: false).End; + Read(dictionary, maxFilters, keep: false).End; /// /// Follows, through , every value decoding the stream describes @@ -137,7 +138,7 @@ public static FilterChainEnd EndOf(PdfDictionary dictionary, int maxFilters) => /// False when ended the reading. public static bool TryFollow(PdfDictionary dictionary, int maxFilters, ref TReader reader) where TReader : struct, IFilterValueReader => - TryRead(dictionary, maxFilters, ref reader, readParameters: true, keep: false, out _); + TryRead(dictionary, maxFilters, ref reader, keep: false, out _); /// Gets the steps decoding runs, in order. public Steps GetEnumerator() => new(this); @@ -197,12 +198,11 @@ public bool MoveNext() /// The stream's dictionary. /// The most filters the stream's data may pass through: the chain ends at the element past them. /// Follows each value to what it leads to, and can end the reading. - /// Whether the parameters are read, or only the names of the filters. /// Whether what decoding needs again is kept, or only what ends the chain. /// What was read. /// False when ended the reading. private static bool TryRead( - PdfDictionary dictionary, int maxFilters, ref TReader reader, bool readParameters, bool keep, out Reading reading) + PdfDictionary dictionary, int maxFilters, ref TReader reader, bool keep, out Reading reading) where TReader : struct, IFilterValueReader { reading = new Reading(); @@ -276,7 +276,7 @@ PdfName name when PdfFilterPipeline.IsImageFilter(name) => FilterChainEnd.Image, Append(ref reading.Names, ref reading.NameCount, filter); } - if (readParameters && (filter == PdfName.FlateDecode || filter == PdfName.LZWDecode)) + if (filter == PdfName.FlateDecode || filter == PdfName.LZWDecode) { if (!parametersRead) { @@ -335,11 +335,11 @@ PdfName name when PdfFilterPipeline.IsImageFilter(name) => FilterChainEnd.Image, /// element past the guard on filters does — the steps before it run, and what they decode is left encoded by the rest /// (#263). /// - private static Reading Read(PdfDictionary dictionary, int maxFilters, bool readParameters, bool keep) + private static Reading Read(PdfDictionary dictionary, int maxFilters, bool keep) { var reader = default(Resolving); - if (!TryRead(dictionary, maxFilters, ref reader, readParameters, keep, out var reading)) + if (!TryRead(dictionary, maxFilters, ref reader, keep, out var reading)) { // The steps read whole before the value stand: the chain ends at the element that value was read for. reading.End = FilterChainEnd.Guard; diff --git a/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs b/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs index aaeed4a1..172967cb 100644 --- a/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs +++ b/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs @@ -69,11 +69,13 @@ public static ReadOnlyMemory Decode(this PdfStream stream, PdfDiagnostics? /// Determines whether the stream's data is an encoded image that leaves alone: whether its /// /Filter entry names an image filter before anything that ends decoding whatever the data holds — a filter /// the library does not know, an element that names no filter, a filter past - /// . + /// , a filter's value or parameter that leads to an object + /// one of the reader's guards cut or left out of the index. /// /// /// None of the data is read: whether a filter before the image filter fails on it, or whether the image is sound, - /// takes decoding it, and reports what it meets. + /// takes decoding it, and reports what it meets. The filters' parameters are read as decoding reads + /// them: one held in an object stream decodes that stream first. /// public static bool HasImageFilter(this PdfStream stream) { diff --git a/tests/AdCodicem.Pdf.Tests/FilterTests.cs b/tests/AdCodicem.Pdf.Tests/FilterTests.cs index e31d6c6e..f9fccca2 100644 --- a/tests/AdCodicem.Pdf.Tests/FilterTests.cs +++ b/tests/AdCodicem.Pdf.Tests/FilterTests.cs @@ -1197,25 +1197,27 @@ public void Gives_a_dictionary_of_parameters_beside_a_chain_to_each_of_its_steps } [Theory] - [InlineData("/Filter /DCTDecode", true)] - [InlineData("/Filter [/ASCIIHexDecode /DCTDecode]", true)] - [InlineData("/Filter [/ASCIIHexDecode /JPXDecode /FlateDecode]", true)] - [InlineData("/Filter [/MadeUpDecode /DCTDecode]", false)] - [InlineData("/Filter [5 /DCTDecode]", false)] - [InlineData("/Filter (DCTDecode)", false)] - [InlineData("/Filter [/ASCIIHexDecode /FlateDecode]", false)] - [InlineData("/Filter null", false)] - [InlineData("", false)] - public void Says_a_stream_is_an_image_only_when_decoding_it_stops_at_an_image_filter(string entries, bool image) + [InlineData("/Filter /DCTDecode", true, false)] + [InlineData("/Filter [/ASCIIHexDecode /DCTDecode]", true, false)] + [InlineData("/Filter [/ASCIIHexDecode /JPXDecode /FlateDecode]", true, false)] + [InlineData("/Filter [/FlateDecode /DCTDecode]", true, true)] + [InlineData("/Filter [/MadeUpDecode /DCTDecode]", false, false)] + [InlineData("/Filter [5 /DCTDecode]", false, false)] + [InlineData("/Filter (DCTDecode)", false, false)] + [InlineData("/Filter [/ASCIIHexDecode /FlateDecode]", false, true)] + [InlineData("/Filter null", false, false)] + [InlineData("", false, false)] + public void Says_a_stream_is_an_image_only_when_decoding_it_stops_at_an_image_filter(string entries, bool image, bool readsParameters) { // Decoding stops at the first filter it does not know, or at an element that names none: the data that comes - // back is not the image a filter after it names. Telling it reads no parameter. + // back is not the image a filter after it names. Telling it reads what decoding reads before it, and no more: the + // parameters of a Flate or LZW step before the chain ends, a value of which a guard cut ends it there (#263). var source = Substitute.For(); var dictionary = new PdfObjectParser(Encoding.ASCII.GetBytes($"<< {entries} /DecodeParms 9 0 R >>"), source: source).ParseObject().AsDictionary().Required(); var stream = new PdfStream(dictionary, PdfStreamData.FromMemory("00>"u8.ToArray())); stream.HasImageFilter().Should().Be(image); - source.ReceivedCalls().Should().BeEmpty(); + source.ReceivedCalls().Should().HaveCount(readsParameters ? 1 : 0); } [Fact] diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index e7b7d23e..e4555a61 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1,4 +1,5 @@ using System.Globalization; +using System.IO.Compression; using System.Text; using AdCodicem.Pdf.Diagnostics; using AdCodicem.Pdf.Documents; @@ -479,4 +480,39 @@ public void A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s(string ke var throwing = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void A_parameter_a_guard_cut_before_an_image_filter_leaves_no_encoded_image(bool guarded) + { + // Object 4's Flate step, before its DCT, reads its parameters in object 20, which MaxObjectLength cuts: decoding ends + // there and gives the Flate data back, which HasImageFilter took for the JPEG (#263). + var jpeg = new byte[] { 0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46, 0x49, 0x46, 0x00, 0xFF, 0xD9 }; + using var output = new MemoryStream(); + + using (var zlib = new ZLibStream(output, CompressionLevel.NoCompression, leaveOpen: true)) + { + zlib.Write(jpeg); + } + + var flate = output.ToArray(); + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + [4] = file.Object( + 4, + string.Create(CultureInfo.InvariantCulture, $"<< /Filter [/FlateDecode /DCTDecode] /DecodeParms [20 0 R null] /Length {flate.Length} >>\nstream\n") + + Encoding.Latin1.GetString(flate) + "\nendstream"), + [20] = file.Object(20, new string(' ', 3000) + "<< /Predictor 1 >>"), + }; + file.Table(21, rows, "/Size 21 /Root 1 0 R"); + using var document = PdfDocument.Open(file.Bytes, guarded ? ObjectGuard : null); + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + stream.HasImageFilter().Should().Be(!guarded); + stream.Decode().ToArray().Should().Equal(guarded ? flate : jpeg); + } } From 7fd8bc48c369071e4f1e37051194606731ca2243 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 09:31:23 +0000 Subject: [PATCH 020/146] fix: report a missing endstream when a guard left the /Length out of the index A /Length that leads to an object a guard left out of the index is unknown, and not judged. With it, the endstream the stream lacked went unreported too: the data was taken where its object's endobj ends it, and neither the diagnostics nor the validator saw a fault. A /Length a guard cut already reported it as the file's fault. As the maintainer settled, both are reported alike now: "The stream's /Length names object 21 0, which one of the reader's limits left out of the index, and no endstream follows its data ...". The object named is the one the chain of references leads to, as for a cut one. A message about a stream's length is now made only when it is reported: a stream parsed again, or one whose length is read once the chain is, made each message and dropped it. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 133 +++++++++++------- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 30 ++++ 2 files changed, 109 insertions(+), 54 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 923a8152..caa554bd 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -756,12 +756,10 @@ private PdfStream ReadStream(PdfDictionary dictionary) { // A length read once the chain is, or one a guard left out of the index or cut, is not judged: nothing is said // of it, nor a message made for it. - if (length != found && declared.Form is not (StreamLengthForm.Deferred or StreamLengthForm.ObjectUnindexed or StreamLengthForm.ObjectCut)) + if (length != found && declared.Form is not (StreamLengthForm.Deferred or StreamLengthForm.ObjectUnindexed or StreamLengthForm.ObjectCut) && + KeepLengthFault(Fault(declared, dataStart, found, found, EndStreamState.Found, null))) { - ReportLengthFault( - PdfDiagnosticCodes.StreamLengthInvalid, - DescribeRecovered(declared, found), - Fault(declared, dataStart, found, found, EndStreamState.Found, null)); + WarnLengthFault(PdfDiagnosticCodes.StreamLengthInvalid, DescribeRecovered(declared, found), dataStart); } _endObjAfterStream = _streamData is null ? null : settled.EndObj; @@ -784,25 +782,26 @@ private PdfStream FinishWithoutEndStream( var stop = search.NextObject is { } next ? next - _baseOffset : endOfData; var fits = length >= 0 && dataStart + (long)length <= stop; var upToStop = (int)Math.Min(stop - dataStart, int.MaxValue); - var where = search.NextObject is { } nextObject - ? string.Create(CultureInfo.InvariantCulture, $"before the next object, at {nextObject}") - : "before the end of the file"; var missing = search.NextObject is null ? EndStreamState.MissingBeforeEndOfFile : EndStreamState.MissingBeforeNextObject; if (!search.Searched) { // The document's searches read as much of the file as they may: what lies past the bytes held is not looked at. var kept = fits ? length : upToStop; - ReportLengthFault( - PdfDiagnosticCodes.StreamLengthInvalid, - fits - ? string.Create( - CultureInfo.InvariantCulture, - $"The stream declared {length} bytes, and no endstream follows them; the declared length is kept without a search, the document's searches for endstream having read as much of the file as they may.") - : string.Create( - CultureInfo.InvariantCulture, - $"{Describe(declared)}, and no endstream follows its data in the bytes read with its object; the {kept} bytes {(search.NextObject is null ? "to the end of the file" : "up to the next object")} are taken as its data without a search, the document's searches for endstream having read as much of the file as they may."), - Fault(declared, dataStart, kept, null, EndStreamState.NotSearched, null)); + + if (KeepLengthFault(Fault(declared, dataStart, kept, null, EndStreamState.NotSearched, null))) + { + WarnLengthFault( + PdfDiagnosticCodes.StreamLengthInvalid, + fits + ? string.Create( + CultureInfo.InvariantCulture, + $"The stream declared {length} bytes, and no endstream follows them; the declared length is kept without a search, the document's searches for endstream having read as much of the file as they may.") + : string.Create( + CultureInfo.InvariantCulture, + $"{Describe(declared)}, and no endstream follows its data in the bytes read with its object; the {kept} bytes {(search.NextObject is null ? "to the end of the file" : "up to the next object")} are taken as its data without a search, the document's searches for endstream having read as much of the file as they may."), + dataStart); + } _endObjUnknown = true; return Finish(dictionary, dataStart, kept, span.Length); @@ -810,12 +809,15 @@ private PdfStream FinishWithoutEndStream( if (fits && search.EndObjAfterDeclared) { - ReportLengthFault( - PdfDiagnosticCodes.StreamTruncated, - string.Create( - CultureInfo.InvariantCulture, - $"The stream has no endstream: its object's endobj follows the {length} bytes it declared, which are taken as its data."), - Fault(declared, dataStart, length, null, EndStreamState.MissingBeforeEndObj, null)); + if (KeepLengthFault(Fault(declared, dataStart, length, null, EndStreamState.MissingBeforeEndObj, null))) + { + WarnLengthFault( + PdfDiagnosticCodes.StreamTruncated, + string.Create( + CultureInfo.InvariantCulture, + $"The stream has no endstream: its object's endobj follows the {length} bytes it declared, which are taken as its data."), + dataStart); + } _endObjAfterStream = EndObjState.Present; return Finish(dictionary, dataStart, length, span.Length); @@ -824,12 +826,16 @@ private PdfStream FinishWithoutEndStream( if (search.ObjectEnd >= 0) { var taken = (int)Math.Min(search.ObjectEnd - _baseOffset - dataStart, int.MaxValue); - ReportLengthFault( - PdfDiagnosticCodes.StreamTruncated, - string.Create( - CultureInfo.InvariantCulture, - $"{Describe(declared)}, and no endstream follows its data {where}: its object's endobj ends it, after {taken} bytes."), - Fault(declared, dataStart, taken, null, EndStreamState.MissingBeforeEndObj, search.NextObject)); + + if (KeepLengthFault(Fault(declared, dataStart, taken, null, EndStreamState.MissingBeforeEndObj, search.NextObject))) + { + WarnLengthFault( + PdfDiagnosticCodes.StreamTruncated, + string.Create( + CultureInfo.InvariantCulture, + $"{Describe(declared)}, and no endstream follows its data {Where(search.NextObject)}: its object's endobj ends it, after {taken} bytes."), + dataStart); + } _endObjAfterStream = EndObjState.Present; return Finish(dictionary, dataStart, taken, span.Length); @@ -837,12 +843,15 @@ private PdfStream FinishWithoutEndStream( if (fits) { - ReportLengthFault( - PdfDiagnosticCodes.StreamLengthInvalid, - string.Create( - CultureInfo.InvariantCulture, - $"The stream declared {length} bytes, and no endstream follows them {where}; the declared length is kept."), - Fault(declared, dataStart, length, null, missing, search.NextObject)); + if (KeepLengthFault(Fault(declared, dataStart, length, null, missing, search.NextObject))) + { + WarnLengthFault( + PdfDiagnosticCodes.StreamLengthInvalid, + string.Create( + CultureInfo.InvariantCulture, + $"The stream declared {length} bytes, and no endstream follows them {Where(search.NextObject)}; the declared length is kept."), + dataStart); + } // Where the object ends is unknown, and its endobj is not judged. _endObjUnknown = true; @@ -851,16 +860,24 @@ private PdfStream FinishWithoutEndStream( // The data runs to the next object, which no declared length passes, or to the end of the file. The object lacks its // endobj, unless what is taken as its data holds one, which may be the object's own: where it ends is then unknown. - ReportLengthFault( - search.NextObject is null ? PdfDiagnosticCodes.StreamTruncated : PdfDiagnosticCodes.StreamLengthInvalid, - string.Create( - CultureInfo.InvariantCulture, - $"{Describe(declared)}, and no endstream follows its data {where}; the {upToStop} bytes {(search.NextObject is null ? "to the end of the file" : "up to that object")} are taken as its data."), - Fault(declared, dataStart, upToStop, null, missing, search.NextObject)); + if (KeepLengthFault(Fault(declared, dataStart, upToStop, null, missing, search.NextObject))) + { + WarnLengthFault( + search.NextObject is null ? PdfDiagnosticCodes.StreamTruncated : PdfDiagnosticCodes.StreamLengthInvalid, + string.Create( + CultureInfo.InvariantCulture, + $"{Describe(declared)}, and no endstream follows its data {Where(search.NextObject)}; the {upToStop} bytes {(search.NextObject is null ? "to the end of the file" : "up to that object")} are taken as its data."), + dataStart); + } _endObjUnknown = search.EndObjSeen; _endObjAfterStream = search.EndObjSeen ? EndObjState.Unknown : EndObjState.Absent; return Finish(dictionary, dataStart, upToStop, span.Length); + + // Where the search stopped, as a report says it. + static string Where(long? nextObject) => nextObject is { } at + ? string.Create(CultureInfo.InvariantCulture, $"before the next object, at {at}") + : "before the end of the file"; } /// @@ -994,6 +1011,12 @@ private readonly DeclaredLength ReadLength(PdfDictionary dictionary) Hint: value.AsInteger() is { } hint and >= 0 and <= int.MaxValue ? (int)hint : null); } + // The object a guard left out of the index, as the one it cut, is the one the chain of references leads to. + if (presence == ObjectPresence.Unindexed) + { + return new DeclaredLength(StreamLengthForm.ObjectUnindexed, Reference: named.Target()); + } + if (presence != ObjectPresence.Defined) { return new DeclaredLength( @@ -1001,7 +1024,6 @@ private readonly DeclaredLength ReadLength(PdfDictionary dictionary) { ObjectPresence.Missing => StreamLengthForm.ObjectMissing, ObjectPresence.Deferred => StreamLengthForm.Deferred, - ObjectPresence.Unindexed => StreamLengthForm.ObjectUnindexed, _ => StreamLengthForm.ObjectUnreadable, }, Reference: reference); @@ -1065,6 +1087,7 @@ private static string Describe(DeclaredLength declared) StreamLengthForm.ObjectMissing => Names(reference, "which the file lacks"), StreamLengthForm.ObjectUnreadable => Names(reference, "which could not be read"), StreamLengthForm.ObjectCut => Names(reference, "which one of the reader's limits cut"), + StreamLengthForm.ObjectUnindexed => Names(reference, "which one of the reader's limits left out of the index"), _ => Names(reference, "which holds " + Value(declared)), }; @@ -1106,26 +1129,28 @@ private readonly StreamLengthFault Fault( }; /// - /// Keeps what was found of the stream being read, and reports it unless a reading of the same stream the reader - /// kept already has: a stream parsed again — after the cache let it go, or the index was rebuilt — is reported once. + /// Keeps what was found of the stream being read, and says whether to report it: unless a reading of the same stream the + /// reader kept already has — a stream parsed again, after the cache let it go or the index was rebuilt, is reported + /// once —, or no diagnostics are kept. The message is made only then. /// - private void ReportLengthFault(string code, string message, StreamLengthFault fault) + private bool KeepLengthFault(StreamLengthFault fault) { // A /Length read once the cross-reference chain is: whether the data ends where it should is known then (#182). One a - // guard left out of the index is unknown, and the guard's report says why. - if (fault.Form is StreamLengthForm.Deferred or StreamLengthForm.ObjectUnindexed) + // guard left out of the index, or cut, is unknown, and the guard's report says why; a missing endstream is still the + // file's (#224), and the only fault reported of it. + if (fault.Form == StreamLengthForm.Deferred) { - return; + return false; } _lengthFault = fault; - - if (_streamData?.IsLengthFaultReported(fault.DataStart) != true) - { - _diagnostics?.Warn(code, message, fault.DataStart); - } + return _diagnostics is not null && _streamData?.IsLengthFaultReported(fault.DataStart) != true; } + /// Reports what was found of the stream being read, whose data starts at . + private readonly void WarnLengthFault(string code, string message, int dataStart) => + _diagnostics!.Warn(code, message, _baseOffset + dataStart); + private PdfStream Finish(PdfDictionary dictionary, int dataStart, int length, int bufferLength) { length = Math.Max(length, 0); diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index e4555a61..54aba73f 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -120,6 +120,36 @@ public void A_length_whose_reference_leads_to_an_object_a_guard_left_out_of_the_ document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitXRefSectionCount); } + [Theory] + [InlineData("21 0 R")] + [InlineData("20 0 R")] + public void A_missing_endstream_is_the_file_s_fault_when_a_guard_left_the_length_out_of_the_index(string length) + { + // Object 4's /Length leads to object 21, which only an older section places, which MaxXRefSectionCount leaves unread: + // the length is unknown, and not judged, but the endstream the stream lacks was not reported either (#224). + var file = new HandWrittenFile(); + var older = new Dictionary { [21] = file.Object(21, "5") }; + var first = file.Table(22, older, "/Size 22"); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, Page), + [4] = file.Object(4, "<< /Length " + length + " >>\nstream\n0 0 m"), + [20] = file.Object(20, "21 0 R"), + }; + file.Table(21, rows, string.Create(CultureInfo.InvariantCulture, $"/Size 22 /Root 1 0 R /Prev {first}")); + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxXRefSectionCount = 1 } }; + using var document = PdfDocument.Open(file.Bytes, options); + + document.GetObject(new PdfObjectId(4)).AsStream().Required().GetRawBytes().ToArray().Should().Equal("0 0 m"u8.ToArray()); + document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.StreamTruncated).Which.Message.Should().StartWith( + "The stream's /Length names object 21 0, which one of the reader's limits left out of the index, and no endstream follows its data"); + document.Reader.TryGetStreamLengthFault(4, out var fault).Should().BeTrue(); + fault.Form.Should().Be(StreamLengthForm.ObjectUnindexed); + fault.Reference.Should().Be(new PdfObjectId(21)); + } + [Theory] [InlineData(false)] [InlineData(true)] From 7fb39ae6037f1be4926ddc512f30d7cd6ec1f43c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 09:37:08 +0000 Subject: [PATCH 021/146] fix: leave to the guard a cross-reference stream whose cut /Length cannot end its data A cross-reference stream whose /Length the chain cannot read takes its data up to the first endstream; with none before the next object or the end of the file, its section is malformed. A /Length that the newer section places, but that MaxObjectLength cuts, waits for the chain the same way, and was judged the same: the fault said no section placed the object, the index was rebuilt, and the validator reported a malformed section. Without the guard the length is read, and the section with it. As the maintainer settled, such a section is the guard's, as one whose /W, /Index or /Filter a guard cut is: it counts as cut, the chain ends there, nothing is rebuilt, and the guard is reached once the chain is read, thrown at opening under ThrowOnLimit. The fault names the reference the guard cut, through a chain of references too. Refs #220, #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 13 +++++++- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 32 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index a40c3832..4c63593f 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3560,10 +3560,21 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect : (DeferredLength?)null; // Data a search found no endstream for, before the next object the sections read so far place or the end of the - // file, ends nowhere the section says: its rows are not read (#220). + // file, ends nowhere the section says: its rows are not read (#220). Where the /Length leads to an object a guard cut, + // the length that would have said is the guard's, as a value its rows are read with is (#263): the section counts as + // cut, the chain ends here, and the guard is reached once the chain is read. if (deferred is { } unended && _endStreamSearches.TryGetValue((xrefStream.Data.Position, true), out var known) && known.Search is { Length: null } search) { + if (Follow(PdfName.Length, dictionary.GetRaw(PdfName.Length), entry: true, out _, out var cutObject) is { } cutFault && + cutObject is { } past) + { + section.Fault = cutFault; + section.CutByLimit = true; + (_chainValuesCut ??= []).Add(past); + return XRefSectionState.Malformed; + } + var where = search.NextObject is { } next ? string.Create(CultureInfo.InvariantCulture, $"before the next object, at {next}") : "before the end of the file"; diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 54aba73f..d749ef6f 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -120,6 +120,38 @@ public void A_length_whose_reference_leads_to_an_object_a_guard_left_out_of_the_ document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitXRefSectionCount); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s(bool chained) + { + // Stream 7 lost its endstream, and its /Length, object 5 — or object 6, which object 5 refers to —, which the newer + // table places, runs past MaxObjectLength: the section was called malformed because no section placed the length, + // and the index rebuilt. Without the guard the length is read, and the section with it: the section is the guard's. + var padded = new string(' ', 3000) + "56"; + var file = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] /Length 5 0 R", + chained ? "6 0 R" : padded, + compressed: false, + predicted: false, + placed: true, + placedToo: chained ? new Dictionary { [6] = padded } : null); + var text = Encoding.Latin1.GetString(file); + var at = text.IndexOf("endstream", StringComparison.Ordinal); + Encoding.ASCII.GetBytes("endstrXam").CopyTo(file, at); + using var document = PdfDocument.Open(file, ObjectGuard); + + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.Reader.Structure.Sections[1].CutByLimit.Should().BeTrue(); + document.Reader.Structure.Sections[1].Fault.Should().EndWith( + string.Create(CultureInfo.InvariantCulture, $"the reference {(chained ? 6 : 5)} 0 R, which one of the reader's limits cut")); + new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().BeFalse(); + + var throwing = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); + throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); + } + [Theory] [InlineData("21 0 R")] [InlineData("20 0 R")] From 080c98fb28d1322df513de8cacd8f70627e1b70f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 09:49:06 +0000 Subject: [PATCH 022/146] fix: report the cut MaxStreamLength makes to the diagnostics a caller passes to Decode Decode promises the diagnostics a caller passes of their own what decoding meets, each time the stream is decoded, and limit.decoded-stream reaches them so. The cut MaxStreamLength makes went to the document's diagnostics alone, once, as the reader read the data: a caller who passed their own got data cut short with no report, and could not tell, the cut being marked only internally. As the maintainer settled, those diagnostics now receive limit.stream-length with each decoding that meets the cut, and the document's still hold it once. A test of ASCII85 data the guard cut added the two lists together and expected the guard once in all: it now finds it once in each, the expectation following the settled rule. The message is made once, as the reader reports it once, rather than each time the data is read past the bound. The XML documentation of Decode and of PdfStreamData.GetBytes says what the guard does to the data and where it is reported. Refs #170 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../IO/Filters/PdfFilterPipeline.cs | 7 +++++ src/AdCodicem.Pdf/IO/PdfFileReader.cs | 21 ++++++++++++--- src/AdCodicem.Pdf/IO/PdfLimitGuard.cs | 7 +++++ src/AdCodicem.Pdf/Objects/PdfStreamData.cs | 10 +++++++ .../Objects/PdfStreamDecodingExtensions.cs | 5 +++- .../AdCodicem.Pdf.Tests/FilterDamageTests.cs | 6 ++++- .../AdCodicem.Pdf.Tests/ReaderLimitsTests.cs | 26 +++++++++++++++++++ 7 files changed, 76 insertions(+), 6 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs index b036c815..a13d95a9 100644 --- a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs +++ b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs @@ -76,6 +76,13 @@ private static ReadOnlyMemory Decode( // its own reports in. if (diagnostics is null || !ReferenceEquals(diagnostics, guard.DocumentDiagnostics)) { + // Data MaxStreamLength cut is reported once in the document's diagnostics, as the reader reads it; diagnostics of + // a caller's own receive it with each decoding, as they receive what decoding meets. + if (diagnostics is not null && stream.Data.CutByGuard && stream.Data.Length > guard.Bound(PdfLimit.StreamLength)) + { + guard.Reach(PdfLimit.StreamLength, diagnostics, guard.DescribeStreamLengthCut(), stream.Data.Position); + } + return DecodeChain(stream, chain, data, diagnostics, guard, out leftEncoded, out cutByGuard); } diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 4c63593f..d002f678 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -4931,6 +4931,22 @@ private void ReachLimit(PdfLimit limit, string what, long position) } } + /// + /// Reaches for the stream whose data starts at , + /// as does: its message is made once, as the stream is reported once, however often its data is + /// read. + /// + private void ReachStreamLength(long position) + { + _guardsReached++; + + if (!_limitsReached.Contains((PdfLimit.StreamLength, position))) + { + _guard.Reach(PdfLimit.StreamLength, _diagnostics, _guard.DescribeStreamLengthCut(), position); + _limitsReached.Add((PdfLimit.StreamLength, position)); + } + } + /// /// Reports, once for each member, that cut member /// of the object stream whose data starts at : its members share that place, and each is @@ -6150,10 +6166,7 @@ public override ReadOnlyMemory GetBytes() { _cutByGuard = true; count = bound; - reader.ReachLimit( - PdfLimit.StreamLength, - $"A stream's data runs past {PdfLimitGuard.FormatLength(bound)}; only what lies within it was read.", - offset); + reader.ReachStreamLength(offset); } if (reader._source.TryLend(offset, count, out var lent)) diff --git a/src/AdCodicem.Pdf/IO/PdfLimitGuard.cs b/src/AdCodicem.Pdf/IO/PdfLimitGuard.cs index 3daaf962..aabebd62 100644 --- a/src/AdCodicem.Pdf/IO/PdfLimitGuard.cs +++ b/src/AdCodicem.Pdf/IO/PdfLimitGuard.cs @@ -68,6 +68,13 @@ public void Reach(PdfLimit limit, PdfDiagnostics? diagnostics, string what, long (diagnostics ?? documentDiagnostics)?.ReachLimit(Code(limit), message, position); } + /// + /// Says what did to a stream's data it cut, as its report starts: the + /// reader's report, once for the stream, and decoding's, in diagnostics a caller passes of its own. + /// + public string DescribeStreamLengthCut() => + $"A stream's data runs past {FormatLength(Bound(PdfLimit.StreamLength))}; only what lies within it was read."; + /// Writes a length the way a message states a bound: 256 MB, 64 KB, 1,000 bytes. public static string FormatLength(int length) => length switch { diff --git a/src/AdCodicem.Pdf/Objects/PdfStreamData.cs b/src/AdCodicem.Pdf/Objects/PdfStreamData.cs index d8db742b..b4487224 100644 --- a/src/AdCodicem.Pdf/Objects/PdfStreamData.cs +++ b/src/AdCodicem.Pdf/Objects/PdfStreamData.cs @@ -32,6 +32,16 @@ public abstract class PdfStreamData internal virtual bool CutByGuard => false; /// Returns the encoded bytes, exactly as they appear in the file. + /// + /// Data read from a document gives no more than bytes: past it, + /// what lies within it comes back, fewer bytes than says, and + /// is reported once in the document's diagnostics. + /// + /// The data was read from a document that is disposed. + /// + /// The data runs past , and its document was opened with + /// . + /// public abstract ReadOnlyMemory GetBytes(); /// Creates stream data over bytes already in memory. diff --git a/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs b/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs index 172967cb..a9433b20 100644 --- a/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs +++ b/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs @@ -42,7 +42,10 @@ public static class PdfStreamDecodingExtensions /// reported once as ; the filters before the last share the /// bound for what they pass along together (). The /// filters after one the bound stopped decode what it kept, as data the reader cut: that it runs out is not - /// reported as damage. A /Filter array that names more filters than + /// reported as damage. So do they data that runs past , of which + /// the reader reads what lies within it, reported as : once in the + /// document's diagnostics, and in diagnostics of your own each time the stream is decoded. A /Filter array that + /// names more filters than /// ends decoding before the first past them, reported as /// when decoding reaches it. /// diff --git a/tests/AdCodicem.Pdf.Tests/FilterDamageTests.cs b/tests/AdCodicem.Pdf.Tests/FilterDamageTests.cs index e02d7e8c..e8df000b 100644 --- a/tests/AdCodicem.Pdf.Tests/FilterDamageTests.cs +++ b/tests/AdCodicem.Pdf.Tests/FilterDamageTests.cs @@ -1252,8 +1252,12 @@ public void Reports_no_missing_greater_than_sign_where_the_reader_cut_ascii85_da using var document = PdfDocument.Open(file, options); Encoding.ASCII.GetString(document.GetObject(new PdfObjectId(3)).AsStream().Required().Decode(diagnostics).Span).Should().Be("Hello World"); + + // The guard is reported once in the document's diagnostics, as the reader reads the data, and in the caller's own, + // as decoding meets the cut. expected = [PdfDiagnosticCodes.LimitStreamLength]; - document.Diagnostics.Select(d => d.Code).Concat(diagnostics.Select(d => d.Code)).Should().Equal(expected); + document.Diagnostics.Select(d => d.Code).Should().Equal(expected); + diagnostics.Select(d => d.Code).Should().Equal(expected); return; } diff --git a/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs b/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs index ebfcbdbb..b9c19f8a 100644 --- a/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs @@ -3748,4 +3748,30 @@ private static byte[] WithIncompleteIndex(byte[] file) var at = trailer + "trailer\n<<".Length; return [.. file.AsSpan(0, at), .. " /Prev 1 0 R"u8, .. file.AsSpan(at)]; } + + [Fact] + public void Decoding_into_diagnostics_of_one_s_own_reports_the_cut_max_stream_length_made_each_time() + { + // Object 4's 2,000 bytes run past MaxStreamLength: the reader reported the cut once, in the document's diagnostics, + // and diagnostics a caller passed to Decode never held it, though limit.decoded-stream reaches them each time. + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + [4] = file.Object(4, "<< /Length 2000 >>\nstream\n" + new string('x', 2000) + "\nendstream"), + }; + file.Table(5, rows, "/Size 5 /Root 1 0 R"); + using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxStreamLength = 1024 } }); + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + var mine = new PdfDiagnostics(); + + stream.Decode(mine).Length.Should().Be(1024); + stream.Decode(mine).Length.Should().Be(1024); + + mine.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitStreamLength, PdfDiagnosticCodes.LimitStreamLength); + mine[0].Message.Should().Be( + "A stream's data runs past 1 KB; only what lies within it was read. Raise PdfReaderLimits.MaxStreamLength to read past it."); + document.Diagnostics.Where(entry => entry.Code == PdfDiagnosticCodes.LimitStreamLength).Should().ContainSingle(); + } } From 00f2f036d7f18e1536d80828515c2f575bd1c914 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 09:49:48 +0000 Subject: [PATCH 023/146] docs: say a document held in memory lends its streams' raw bytes A stream's raw bytes taken from a document opened over memory are a view of that memory, not a copy: they keep it alive after Dispose, for as long as they are held, and change as it does. Refs #170 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/Documents/PdfDocument.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/AdCodicem.Pdf/Documents/PdfDocument.cs b/src/AdCodicem.Pdf/Documents/PdfDocument.cs index 4f56f3b0..bd38164f 100644 --- a/src/AdCodicem.Pdf/Documents/PdfDocument.cs +++ b/src/AdCodicem.Pdf/Documents/PdfDocument.cs @@ -27,7 +27,9 @@ namespace AdCodicem.Pdf.Documents; /// accessor that resolves, a stream's data asked for —, however it was opened and whether or not it was read before. /// What was handed out stays the caller's: a dictionary's direct values, a stream's , /// every object's ToString. Disposing lets go of what the document kept: the objects it read, its index, and the -/// file, closed when it was opened from a path, and held no longer when it was opened from memory or a stream. +/// file, closed when it was opened from a path, and held no longer when it was opened from memory or a stream. A stream's +/// raw bytes taken before from a document held in memory are a view of that memory rather than a copy +/// (): for as long as they are held, they keep it alive, and they change as it does. /// /// public sealed class PdfDocument : IDisposable From b1a2dc83fee580d48b80feec1298500b7325b5ef Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 09:53:42 +0000 Subject: [PATCH 024/146] test: hold that telling an image reaches a guard under ThrowOnLimit as decoding does HasImageFilter now reads the parameters decoding reads before the image filter, and loading one past MaxObjectLength reaches the guard: under ThrowOnLimit, the caller is told so, as Decode would tell them. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index d749ef6f..c0a063ac 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -576,5 +576,14 @@ public void A_parameter_a_guard_cut_before_an_image_filter_leaves_no_encoded_ima stream.HasImageFilter().Should().Be(!guarded); stream.Decode().ToArray().Should().Equal(guarded ? flate : jpeg); + + // Loading the parameter reaches the guard, as decoding would: a caller who asked for it is told so. + if (guarded) + { + using var throwing = PdfDocument.Open(file.Bytes, ObjectGuard with { ThrowOnLimit = true }); + var image = throwing.GetObject(new PdfObjectId(4)).AsStream().Required(); + image.Invoking(value => value.HasImageFilter()).Should().Throw() + .Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); + } } } From 123ddcfa442a338b69791c005c5e488066c1f7e4 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 09:53:42 +0000 Subject: [PATCH 025/146] docs: say what loading a value the filters read can throw from Decode and HasImageFilter Decoding loads the values its filters read, and so can reach a guard under ThrowOnLimit, or meet a disposed document: the exceptions of Decode and HasImageFilter now say so. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../Objects/PdfStreamDecodingExtensions.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs b/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs index a9433b20..6149b80f 100644 --- a/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs +++ b/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs @@ -58,10 +58,12 @@ public static class PdfStreamDecodingExtensions /// /// Decoding reaches one of its document's guards — the encoded data runs past /// , the data decodes past - /// , or decoding reaches the filter past - /// —, and the document was opened with + /// , decoding reaches the filter past + /// , or loading a value the filters read reaches one, + /// among them —, and the document was opened with /// . /// + /// The stream was read from a document that is disposed. public static ReadOnlyMemory Decode(this PdfStream stream, PdfDiagnostics? diagnostics = null) { ArgumentNullException.ThrowIfNull(stream); @@ -80,6 +82,11 @@ public static ReadOnlyMemory Decode(this PdfStream stream, PdfDiagnostics? /// takes decoding it, and reports what it meets. The filters' parameters are read as decoding reads /// them: one held in an object stream decodes that stream first. /// + /// + /// Loading a value the filters read reaches one of its document's guards, and the document was opened with + /// . + /// + /// A value the filters read is to be loaded from a document that is disposed. public static bool HasImageFilter(this PdfStream stream) { ArgumentNullException.ThrowIfNull(stream); From 54d963b40713ec36c82b769de501ef8d2782010f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 10:03:35 +0000 Subject: [PATCH 026/146] test: hold the paths of a stream's end that no test reached The batch's patch left lines of where a stream's data ends that no test reached. Each now has a file that reaches it: - an endobj that ends a longer token, read in one read or parted between two, or right where the data starts; - a stream parsed alone, in a window that is not the end of the data, with no endstream in it; - a container skipped past the depth the parser follows, which the next object's header ends; - a comment in the gap after a length that runs past the window and the first read after it, and the check the reader keeps for the stream's next reading; - "endstreamx" read on past the window, and white space read on to the end of the file; - a stream left unsearched whose declared length runs past the end of the file. The construction of streams nested in one another's dictionaries is shared by two tests now. Refs #174 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- tests/AdCodicem.Pdf.Tests/ParserTests.cs | 33 ++++ .../AdCodicem.Pdf.Tests/StreamLengthTests.cs | 160 +++++++++++++++--- tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs | 28 +++ 3 files changed, 197 insertions(+), 24 deletions(-) diff --git a/tests/AdCodicem.Pdf.Tests/ParserTests.cs b/tests/AdCodicem.Pdf.Tests/ParserTests.cs index de588d43..b9a0dad4 100644 --- a/tests/AdCodicem.Pdf.Tests/ParserTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ParserTests.cs @@ -183,6 +183,9 @@ public void Ends_the_containers_an_object_left_open_at_the_next_object_s_header_ [InlineData("hello\nendobj\n% a note\n6 0 obj\n", 5)] [InlineData("hello\nendobj % note endobj\n6 0 obj\n", 5)] [InlineData("hello\nendobj\n% a note\nx\n6 0 obj\n", -1)] + [InlineData("hello endobjx\n6 0 obj\n", -1)] + [InlineData("helloendobj\n6 0 obj\n", -1)] + [InlineData("endobj\n6 0 obj\n", 0)] public void Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow(string data, int length) { // Parsed alone, a stream with no endstream ends where its object's endobj ends it, a comment after which is white @@ -194,6 +197,36 @@ public void Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_an value.AsStream().Required().RawLength.Should().Be(length < 0 ? data.IndexOf("6 0 obj", StringComparison.Ordinal) : length); } + [Fact] + public void Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one() + { + // Parsed alone in a window that is not the end of the data, a stream whose endstream and next header lie past it + // asks for a larger window, which may show them. + var parser = new PdfObjectParser(Encoding.ASCII.GetBytes("4 0 obj\n<< /Length 200 >>\nstream\nabc"), endsData: false); + + parser.TryReadIndirectObject(out _, out _); + + parser.IsTruncated.Should().BeTrue(); + } + + [Theory] + [InlineData("[", "an array")] + [InlineData("<< /A ", "a dictionary")] + public void Ends_a_container_skipped_for_its_depth_at_the_next_object_s_header(string opening, string what) + { + // Past the depth the parser follows, the container is skipped by counting brackets: the next object's header ends it, + // as it ends one the parser follows (#230). + var text = string.Concat(Enumerable.Repeat(opening, 130)) + "1\n6 0 obj\n"; + var diagnostics = new PdfDiagnostics(); + var parser = new PdfObjectParser(Encoding.ASCII.GetBytes(text), diagnostics: diagnostics); + + _ = parser.ParseObject(); + + diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.SyntaxDepthExceeded, PdfDiagnosticCodes.SyntaxTruncatedObject); + diagnostics[1].Message.Should().Be( + $"An object header came inside {what}, which was never closed; the 128 arrays or dictionaries around it were never closed either."); + } + [Theory] [InlineData("<< /A 1\n6")] [InlineData("<< /A 1\n6 0")] diff --git a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs index a239679b..a24aafea 100644 --- a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs +++ b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs @@ -1548,30 +1548,7 @@ public void Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of // it a few times over together; then a stream keeps its declared length without a search, and says so. const int Streams = 30; const int Declared = 100_000; - var nested = Invariant($"{Streams + 2} 0 obj<>stream\n"); - - for (var number = Streams + 1; number >= 3; number--) - { - nested = Invariant($"{number} 0 obj<>stream\n"); - } - - var head = "%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n" + nested; - var offsets = new List { 0, 9, head.IndexOf("2 0 obj", StringComparison.Ordinal) }; - - for (var number = 3; number <= Streams + 2; number++) - { - offsets.Add(head.IndexOf(Invariant($"{number} 0 obj<<"), StringComparison.Ordinal)); - } - - var body = head + new string('x', 1024 * 1024) + "\n"; - var xref = new StringBuilder(Invariant($"xref\n0 {offsets.Count}\n0000000000 65535 f\r\n")); - - foreach (var offset in offsets.Skip(1)) - { - xref.Append(Invariant($"{offset:D10} 00000 n\r\n")); - } - - var file = Encoding.Latin1.GetBytes(body + xref + Invariant($"trailer\n<< /Size {offsets.Count} /Root 1 0 R >>\nstartxref\n{body.Length}\n%%EOF\n")); + var (file, head) = NestedStreams(Streams, Declared); using var source = new StrictCountingSource(file); using var document = PdfDocument.Open(source, options: null, ownsSource: false); @@ -2179,4 +2156,139 @@ private static byte[] OneStream(string body, bool endObj = true) file.Table(7, rows, "/Size 7 /Root 1 0 R"); return file.Bytes; } + + /// + /// A document of streams, objects 3 on, each declaring bytes, whose + /// headers each lie inside the dictionary of the one before, as a string, followed by a megabyte of data no endstream + /// ends; and the text before that megabyte. + /// + private static (byte[] File, string Head) NestedStreams(int streams, int declared) + { + var nested = Invariant($"{streams + 2} 0 obj<>stream\n"); + + for (var number = streams + 1; number >= 3; number--) + { + nested = Invariant($"{number} 0 obj<>stream\n"); + } + + var head = "%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n" + nested; + var offsets = new List { 0, 9, head.IndexOf("2 0 obj", StringComparison.Ordinal) }; + + for (var number = 3; number <= streams + 2; number++) + { + offsets.Add(head.IndexOf(Invariant($"{number} 0 obj<<"), StringComparison.Ordinal)); + } + + var body = head + new string('x', 1024 * 1024) + "\n"; + var xref = new StringBuilder(Invariant($"xref\n0 {offsets.Count}\n0000000000 65535 f\r\n")); + + foreach (var offset in offsets.Skip(1)) + { + xref.Append(Invariant($"{offset:D10} 00000 n\r\n")); + } + + var file = Encoding.Latin1.GetBytes(body + xref + Invariant($"trailer\n<< /Size {offsets.Count} /Root 1 0 R >>\nstartxref\n{body.Length}\n%%EOF\n")); + + return (file, head); + } + + [Fact] + public void A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds() + { + // As streams whose data share a stretch exhaust what the searches may read, the last is left unsearched; its declared + // length runs past the end of the file, and the data up to it is taken, without a search, and said so. + const int Streams = 30; + const int Declared = 10_000_000; + var (file, _) = NestedStreams(Streams, Declared); + using var document = PdfDocument.Open(file); + + for (var number = 3; number <= Streams + 2; number++) + { + document.GetObject(new PdfObjectId(number)); + } + + document.Reader.TryGetStreamLengthFault(Streams + 2, out var last).Should().BeTrue(); + last.EndStream.Should().Be(EndStreamState.NotSearched); + last.Taken.Should().Be((int)(file.Length - last.DataStart)); + document.Diagnostics.Should().ContainSingle(d => d.Position == last.DataStart).Which.Message.Should().Be( + Invariant($"The stream declared {Declared} bytes, and no endstream follows its data in the bytes read with its object; the {last.Taken} bytes to the end of the file are taken as its data without a search, the document's searches for endstream having read as much of the file as they may.")); + } + + [Fact] + public void A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading() + { + // The declared length ends 20 bytes short of the window's edge, and a comment after it runs past the edge and past the + // first read after it: the reader reads on through it to the endstream, and keeps what it found for the stream's next + // reading, after the cache let the stream go. + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + }; + rows[4] = file.Position; + const string Before = "4 0 obj\n<< /Length 0000 >>\nstream\n"; + var length = PdfFileReader.InitialObjectWindow - 20 - Before.Length; + file.Raw(Before.Replace("0000", Invariant($"{length:D4}"), StringComparison.Ordinal) + new string('a', length) + + "\n% " + new string('c', 200) + "\nendstream\nendobj\n"); + rows[6] = file.Object(6, "(six)"); + file.Table(7, rows, "/Size 7 /Root 1 0 R"); + using var document = PdfDocument.Open(file.Bytes, PdfReaderOptions.Default with { ObjectCacheCapacity = 1 }); + + document.GetObject(new PdfObjectId(4)).AsStream().Required().RawLength.Should().Be(length); + document.GetObject(new PdfObjectId(6)); + document.GetObject(new PdfObjectId(4)).AsStream().Required().RawLength.Should().Be(length); + document.Diagnostics.Should().BeEmpty(); + document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); + } + + [Fact] + public void An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj() + { + // White space after the declared length runs past the window's edge and the first read after it, to "endstreamx": + // read as the parser reads what follows a stream, the length is confirmed, and its object lacks its endobj. + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + }; + rows[4] = file.Position; + const string Before = "4 0 obj\n<< /Length 0000 >>\nstream\n"; + var length = PdfFileReader.InitialObjectWindow - 20 - Before.Length; + file.Raw(Before.Replace("0000", Invariant($"{length:D4}"), StringComparison.Ordinal) + new string('a', length) + + "\n" + new string(' ', 100) + "endstreamx\nendobj\n"); + rows[6] = file.Object(6, "(six)"); + file.Table(7, rows, "/Size 7 /Root 1 0 R"); + using var document = PdfDocument.Open(file.Bytes); + + document.GetObject(new PdfObjectId(4)).AsStream().Required().RawLength.Should().Be(length); + document.Reader.IsEndObjMissing(4, out _).Should().BeTrue(); + } + + [Theory] + [InlineData("", false)] + [InlineData("endstream\nendobj", true)] + public void White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file(string end, bool ended) + { + // A file with no index, whose last object is a stream of 9,000 bytes followed by white space past the first read after + // it — to the end of the file, or to an endstream and an endobj that end it. + const int Length = 9000; + var text = "%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n" + + Invariant($"4 0 obj\n<< /Length {Length} >>\nstream\n") + new string('a', Length) + new string(' ', 200) + end; + using var document = PdfDocument.Open(Encoding.ASCII.GetBytes(text)); + + document.GetObject(new PdfObjectId(4)).AsStream().Required().RawLength.Should().Be(Length); + + if (ended) + { + document.Diagnostics.Should().NotContain(entry => entry.Code.StartsWith("stream.", StringComparison.Ordinal)); + document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); + } + else + { + document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.StreamLengthInvalid).Which.Message.Should().Be( + "The stream declared 9000 bytes, and no endstream follows them before the end of the file; the declared length is kept."); + } + } } diff --git a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs index a20a224f..7bcc3491 100644 --- a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs +++ b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs @@ -414,6 +414,34 @@ public void An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_pa .Which.Message.Should().EndWith(string.Create(CultureInfo.InvariantCulture, $"its object's endobj ends it, after {length} bytes.")); } + [Theory] + [InlineData(-7)] + [InlineData(-5)] + [InlineData(-3)] + [InlineData(-1)] + public void An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it(int shift) + { + // As above, but the keyword ends the token "aendobj", parted between the window and the first read past it: no endobj + // ends the data, and the declared length, which ends before the next object, is kept. + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + }; + rows[4] = file.Position; + const string Before = "4 0 obj\n<< /Length 3 >>\nstream\n"; + var length = Window + shift - Before.Length; + file.Raw(Before + new string('a', length) + "endobj\n% c\n7 0 obj\n(stray)\nendobj\n"); + rows[6] = file.Object(6, "(six)"); + file.Table(7, rows, "/Size 7 /Root 1 0 R"); + using var document = PdfDocument.Open(file.Bytes); + + document.GetObject(new PdfObjectId(4)).AsStream().Required().RawLength.Should().Be(3); + document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.StreamLengthInvalid) + .Which.Message.Should().EndWith("the declared length is kept."); + } + [Fact] public void An_object_whose_header_lies_past_the_window_is_read_where_the_index_says() { From 3dd9d0a5cef6699c5cd50276a1d1b762780d4311 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 10:03:36 +0000 Subject: [PATCH 027/146] refactor: drop a check no position can meet where a deferred length's endstream is found The place a deferred cross-reference stream's endstream is looked for after is where its data starts plus the bytes the chain took, never before the start of the file. Refs #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index d002f678..6bab9e56 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -2221,7 +2221,7 @@ private void ReachChainValuesCut() /// private long EndStreamAfter(long position) { - var at = position < 0 ? -1 : NextSignificant(position); + var at = NextSignificant(position); Span keyword = stackalloc byte[StreamEnds.EndStreamKeyword.Length]; var read = at < 0 || at >= _source.Length ? 0 : _source.Read(at, keyword[..(int)Math.Min(keyword.Length, _source.Length - at)]); return keyword[..read].SequenceEqual(StreamEnds.EndStreamKeyword) ? at : position; From cf7b1dfca3ba0e9e2579b3d329ff7fefacfa2603 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 10:07:11 +0000 Subject: [PATCH 028/146] test: hold how a stream's end is read once the searches read as much as they may Once the document's searches for endstream have read as much of the file as they may, white space after a declared length that runs past the window is not read on. A length the first read past the window cannot confirm is kept without a search, and one whose endstream that read ends after leaves what follows the keyword unseen. Neither path had a test. Refs #174 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../AdCodicem.Pdf.Tests/StreamLengthTests.cs | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs index a24aafea..6c55bf88 100644 --- a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs +++ b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs @@ -2160,9 +2160,9 @@ private static byte[] OneStream(string body, bool endObj = true) /// /// A document of streams, objects 3 on, each declaring bytes, whose /// headers each lie inside the dictionary of the one before, as a string, followed by a megabyte of data no endstream - /// ends; and the text before that megabyte. + /// ends, then by , objects numbered on from the last stream; and the text before that megabyte. /// - private static (byte[] File, string Head) NestedStreams(int streams, int declared) + private static (byte[] File, string Head) NestedStreams(int streams, int declared, params string[] after) { var nested = Invariant($"{streams + 2} 0 obj<>stream\n"); @@ -2180,6 +2180,13 @@ private static (byte[] File, string Head) NestedStreams(int streams, int declare } var body = head + new string('x', 1024 * 1024) + "\n"; + + foreach (var text in after) + { + offsets.Add(body.Length); + body += text; + } + var xref = new StringBuilder(Invariant($"xref\n0 {offsets.Count}\n0000000000 65535 f\r\n")); foreach (var offset in offsets.Skip(1)) @@ -2291,4 +2298,36 @@ public void White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_ "The stream declared 9000 bytes, and no endstream follows them before the end of the file; the declared length is kept."); } } + + [Fact] + public void A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window() + { + // Once streams whose data share a stretch have read as much of the file as the searches may, white space after a + // declared length that runs past the window is not read on: a length the first read after the window cannot confirm + // is kept without a search, and one whose endstream that read ends after leaves what follows the keyword unseen. + const int Streams = 30; + var unread = Gapped(Streams + 3, new string(' ', 200) + "endstream\nendobj\n", out var length); + var unseen = Gapped(Streams + 4, new string(' ', 50) + "endstream endobj\n", out _); + var (file, _) = NestedStreams(Streams, 100_000, unread, unseen); + using var document = PdfDocument.Open(file); + + for (var number = 3; number <= Streams + 2; number++) + { + document.GetObject(new PdfObjectId(number)); + } + + document.GetObject(new PdfObjectId(Streams + 3)).AsStream().Required().RawLength.Should().Be(length); + document.Reader.TryGetStreamLengthFault(Streams + 3, out var fault).Should().BeTrue(); + fault.EndStream.Should().Be(EndStreamState.NotSearched); + document.GetObject(new PdfObjectId(Streams + 4)).AsStream().Required().RawLength.Should().Be(length); + document.Reader.TryGetStreamLengthFault(Streams + 4, out _).Should().BeFalse(); + + // An object whose declared length ends 20 bytes short of its window's edge, followed by what follows it. + static string Gapped(int number, string after, out int length) + { + var head = Invariant($"{number} 0 obj\n<< /Length 0000 >>\nstream\n"); + length = PdfFileReader.InitialObjectWindow - 20 - head.Length; + return head.Replace("0000", Invariant($"{length:D4}"), StringComparison.Ordinal) + new string('a', length) + after; + } + } } From c28d8027e59da447c44acd363ebd8a0a2526b390 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 10:09:02 +0000 Subject: [PATCH 029/146] test: hold the retake of a deferred length by a row with no type field A cross-reference stream whose /W gives its type field no width makes every row of type 1. The section's own row for its /Length, before rows that spell an endstream, then places it, and the data is taken again to the length it gives. No test had such a section. Refs #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 28 +++++++++++++++++++ .../DocumentReaderTests.cs | 12 ++++++++ 2 files changed, 40 insertions(+) diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index c823b54c..ca1d3c22 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -212,6 +212,34 @@ public static byte[] AloneWithIndirectLengthBeforeSpelledRows(bool reservedRowFi return file.ToArray(); } + /// + /// A file of one cross-reference stream, object 5, whose rows have no type field — /W [0 4 2], every row of type 1 + /// —, whose /Length names object 4, which only that stream indexes, and whose rows of objects 6 and 7 spell + /// endstream after an end-of-line, before the page's row. The file is valid. + /// + public static byte[] AloneWithIndirectLengthAndNoTypeField() + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + file.Object(4, "42"); + var stream = file.Position; + var data = new List(); + + foreach (var offset in new[] { file.OffsetOf(1), file.OffsetOf(2), file.OffsetOf(4), stream }) + { + data.AddRange([(byte)(offset >> 24), (byte)(offset >> 16), (byte)(offset >> 8), (byte)offset, 0, 0]); + } + + data.AddRange("\nendstream\0\0"u8.ToArray()); + var page = file.OffsetOf(3); + data.AddRange([(byte)(page >> 24), (byte)(page >> 16), (byte)(page >> 8), (byte)page, 0, 0]); + file.Stream(5, "/Type /XRef /Size 8 /W [0 4 2] /Index [1 2 4 2 6 2 3 1] /Root 1 0 R /Length 4 0 R", [.. data]); + file.StartXRef(stream); + return file.ToArray(); + } + /// /// A file of two revisions. The first writes object 8 as (old) and indexes it in cross-reference stream 7. The /// second writes it again as (new), and indexes it in cross-reference stream 10, whose /Length names diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 023b754a..92947586 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -1869,6 +1869,18 @@ public void A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().BeFalse(); } + [Fact] + public void A_deferred_length_is_taken_again_by_a_row_with_no_type_field() + { + // With no type field, every row is of type 1: the section's row for its /Length, before rows that spell an endstream, + // places it, and the data is taken again to the length it gives (#220). + using var document = PdfDocument.Open(ChainFiles.AloneWithIndirectLengthAndNoTypeField()); + + document.Reader.Structure.Sections.Should().ContainSingle().Which.Incomplete.Should().BeFalse(); + document.GetObject(new PdfObjectId(3)).AsDictionary().IsOfType(PdfName.Page).Should().BeTrue(); + document.Diagnostics.Should().NotContain(entry => entry.Code == PdfDiagnosticCodes.StreamLengthInvalid); + } + [Fact] public void A_deferred_length_is_not_checked_against_data_a_guard_cut() { From d27cb45db9aac56acdad6c1c96c541427a37c384 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 10:10:37 +0000 Subject: [PATCH 030/146] test: hold that a deferred length a newer section refused takes no data again A newer table that refuses its row for the object a cross-reference stream's /Length names leaves no row of that stream standing for it: the data stays as the chain took it, up to the endstream its rows spell, and the section is said to lack the rows past it. No test had such a chain. Refs #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../DocumentReaderTests.cs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 92947586..f2728814 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -1881,6 +1881,26 @@ public void A_deferred_length_is_taken_again_by_a_row_with_no_type_field() document.Diagnostics.Should().NotContain(entry => entry.Code == PdfDiagnosticCodes.StreamLengthInvalid); } + [Fact] + public void A_deferred_length_a_newer_section_refused_takes_no_data_again() + { + // A third revision's table refuses the row it gives object 4, the update's /Length: no row of the update's stream + // stands for it then, and the data is left as the chain took it, up to the endstream its rows spell (#220). + var file = ChainFiles.UpdateWithIndirectLength(spellingEndStream: true); + var text = Encoding.Latin1.GetString(file); + var startxref = text.LastIndexOf("startxref\n", StringComparison.Ordinal) + "startxref\n".Length; + var second = text[startxref..text.IndexOf('\n', startxref)]; + var update = string.Create( + CultureInfo.InvariantCulture, + $"xref\n4 1\n0000000010 99999 n\r\ntrailer\n<< /Size 11 /Root 1 0 R /Prev {second} >>\nstartxref\n{file.Length}\n%%EOF\n"); + using var document = PdfDocument.Open(file.Concat(Encoding.ASCII.GetBytes(update)).ToArray()); + + var sections = document.Reader.Structure.Sections; + sections[0].RefusedRow.Should().NotBeNull(); + sections[1].Incomplete.Should().BeTrue(); + sections[1].Fault.Should().Be("it holds 1 row where its /Index and /Size declare 5"); + } + [Fact] public void A_deferred_length_is_not_checked_against_data_a_guard_cut() { From e1452f609ffa515eb58297f329382819b5e80f96 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 10:14:04 +0000 Subject: [PATCH 031/146] refactor: keep what a guard's cut left of an object by whether it was read alone A read past MaxObjectLength gives what the cut left of the object when it read one. A header that names another object, or a stream, has already returned by then, so the conditions that tested them again there could not fail. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 6bab9e56..36c8f93d 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -4795,9 +4795,10 @@ private bool TryReadUnrecorded(PdfObjectId id, long offset, out PdfObject value, { if (windowSize >= maxWindow) { - // What the cut left of the object, which a caller may take as a place to look at, never as its value. + // What the cut left of the object, which a caller may take as a place to look at, never as its value: a + // header read whole that names another object, or a stream, returned above. pastGuard = true; - value = read && found == id && parsed is not PdfStream ? parsed : PdfNull.Instance; + value = read ? parsed : PdfNull.Instance; return false; } From 18c3b3e8dfce0076b3891e954e919f4a7ab65305 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 10:14:15 +0000 Subject: [PATCH 032/146] docs: bring the documents in line with the stream-end and guard rules of the first review ADR 34, the architecture, the threat model, M02 and the concepts, guides and references follow the rules the first review settled: comments in the gap and after endobj, a guard-cut or unindexed /Length, a /Prev or /XRefStm through a cut object, a cross-reference stream's cut /Length, HasImageFilter's parameters, limit.stream-length in a caller's diagnostics, and the header at a window's edge. Refs #174, #224, #230, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- ...-is-readable-and-the-readers-guards-are.md | 74 +++++++++++-------- docs/architecture.md | 36 ++++++--- docs/milestones/M02.md | 34 +++++---- docs/threat-model.md | 33 +++++---- docs/website/docs/concepts/diagnostics.md | 30 +++++--- docs/website/docs/concepts/lazy-reading.md | 44 ++++++----- docs/website/docs/concepts/reader-limits.md | 7 +- docs/website/docs/concepts/validation.md | 4 +- .../guides/find-what-the-reader-repaired.md | 4 + .../website/docs/guides/tune-reader-memory.md | 7 +- docs/website/docs/reference/diagnostics.md | 38 ++++++---- docs/website/docs/reference/object-model.md | 11 ++- docs/website/docs/reference/reader-limits.md | 32 +++++--- .../docs/reference/validation-rules.md | 9 ++- 14 files changed, 218 insertions(+), 145 deletions(-) diff --git a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md index 08c6c901..23f155f2 100644 --- a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md +++ b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md @@ -90,29 +90,38 @@ handed back data no filter decodes — none named, an image filter, one the libr none — whole, past `MaxDecodedStreamLength` and unreported, and a `/Length` past what an array holds threw an `OutOfMemoryException` each time the data was read. The data is now kept by nothing and read each time it is asked for, so the guard bounds each read: past it the data is cut, what the filters lose past the cut is no damage, the stream is -reported once, and its length stays the one the file holds. `MaxDecodedStreamLength` keeps bounding what the filters -produce, so an uncompressed image past the default needs the new guard raised alone. Nor does a stream's data count -toward `MaxObjectLength`: a `/Length` the file cannot hold grew the window an object is parsed through up to that bound, -reading 21.5 MB to find five bytes and reporting `limit.object`; no window grows to find an `endstream` now, and one -search past the window does. What that search reads, with the white space read on after a declared length, is bounded -by `EndStreamSearchPasses`, four times the file for a document's searches together, which stays an internal constant -with its reason rewritten: a valid file is read past the parser's window only by the search for the data of a cross-reference -stream whose `/Length` the chain cannot read, and by white space and comments after a declared length that run past a -first read — about twice the file at most. `MaxEndStreamGap`, the four bytes of white space accepted between a stream's -data and its `endstream`, is removed: more, or a comment, made a right `/Length` a wrong one, and white space and -comments are read on as far as they run, counted with the searches. `EndObjLookahead` is the size of that first read, no -longer a bound. `MaxTrailerLength` bounds a cross-reference stream's dictionary alone: the data of one whose `/Length` the -chain cannot read is searched for its `endstream` in the file rather than through a window the guard bounded — a stream -of 70,000 bytes of rows reads as its twin with a direct `/Length`, with no `limit.trailer`, and `ThrowOnLimit` no longer -throws for it —, and data no `endstream` follows before the next object the sections read so far place, or the end of -the file, makes its section malformed: the file's fault, and no guard's. A value read through a reference to an object a -guard cut or left out of the index — a stream's `/Length`, `/Filter` or `/DecodeParms`, a value of a filter's parameters, -a cross-reference stream's `/W`, `/Index` or `/Size` — is the guard's, not the file's: an integer the cut parted, `1` of -`15`, is only a place to look for `endstream` at, never a declared length; a chain of filters ends there, its data left -encoded; and a cross-reference section whose values lead to an object `MaxObjectLength` cut ends the chain, nothing -rebuilt. A guard that loading a value of a stream's filters reaches is that value's, not a cut of the stream's own data, -whose faults it hid. No bound is added for those values, nor for the next object's header that ends an object's open -containers: no valid object holds `obj` where a value is expected. +reported once in its document's diagnostics and, as `limit.decoded-stream` is, with each decoding into diagnostics a +caller passes to `Decode`, and its length stays the one the file holds. `MaxDecodedStreamLength` keeps bounding what the +filters produce, so an uncompressed image past the default needs the new guard raised alone. Nor does a stream's data +count toward `MaxObjectLength`: a `/Length` the file cannot hold grew the window an object is parsed through up to that +bound, reading 21.5 MB to find five bytes and reporting `limit.object`; no window grows to find an `endstream` now, and +one search past the window does. What that search reads, with the white space read on after a declared length, is +bounded by `EndStreamSearchPasses`, four times the file for a document's searches together, which stays an internal +constant with its reason rewritten: a valid file is read past the parser's window only by the search for the data of a +cross-reference stream whose `/Length` the chain cannot read, and by white space and comments after a declared length +that run past a first read — about twice the file at most. `MaxEndStreamGap`, the four bytes of white space accepted +between a stream's data and its `endstream`, is removed: more, or a comment, made a right `/Length` a wrong one, and +white space and comments are read on as far as they run, counted with the searches — a comment only once white space +comes before it, since a `%` right where the length ends is the data's. Comments after the `endobj` that ends a stream +with no `endstream` are white space too, read through the search's own reads: neither rule adds a bound. +`EndObjLookahead` is the size of that first read, no longer a bound. `MaxTrailerLength` bounds a cross-reference +stream's dictionary alone: the data of one whose `/Length` the chain cannot read is searched for its `endstream` in the +file rather than through a window the guard bounded — a stream of 70,000 bytes of rows reads as its twin with a direct +`/Length`, with no `limit.trailer`, and `ThrowOnLimit` no longer throws for it —, and data no `endstream` follows before +the next object the sections read so far place, or the end of the file, makes its section malformed: the file's fault, +and no guard's, unless that `/Length` leads to an object `MaxObjectLength` cut (below). A value read through a reference +to an object a guard cut or left out of the index — a stream's `/Length`, `/Filter` or `/DecodeParms`, a value of a +filter's parameters, a cross-reference stream's `/W`, `/Index` or `/Size`, a trailer's `/Prev` or `/XRefStm` — is the +guard's, not the file's: an integer the cut parted, `1` of `15`, is only a place to look for `endstream` at, never a +declared length, though a stream that lacks its `endstream` is still the file's fault; a chain of filters ends there, +its data left encoded; and a cross-reference section whose values lead to an object `MaxObjectLength` cut ends the +chain, nothing rebuilt, the older sections and their trailers the guard's to hide, and the guard reached once the chain +is read. So does a cross-reference stream's `/Length` the chain could not read that leads to such an object, wherever +only that length could say how far the data runs: no `endstream` comes before the next object or the end of the file, or +the rows spell one and none confirms what the cut left of the length, the rows past it then the guard's. A guard that +loading a value of a stream's filters reaches is that value's, not a cut of the stream's own data, whose faults it hid. +No bound is added for those values, nor for the next object's header that ends an object's open containers: no valid +object holds `obj` where a value is expected. ## Context @@ -174,20 +183,23 @@ guard: active by default, reported when it is reached, and lifted by an option. value that leads to an object a guard cut or left out of the index (amended on 2026-10-23). An object, a section or a trailer is reported once, however often it is read again; a stream, by the first of its filters that stopped there, each time it is decoded past the bound into diagnostics a caller supplies, and once in its document's own, however - often it is decoded (amended on 2026-10-15); a stream's data `MaxStreamLength` cut, once for each stream (amended on - 2026-10-23); a full index, once a document. The filters after one a guard stopped, and those of data `MaxStreamLength` - cut, are given data the reader cut: that it runs out — a lost tail, a missing checksum, a digit without its pair, a - row the predictor cannot finish — is not reported, and a fault in what was read is. The code `filter.limit-exceeded`, added by T31 and never released, becomes `limit.decoded-stream`. + often it is decoded (amended on 2026-10-15); a stream's data `MaxStreamLength` cut, with each decoding into + diagnostics a caller supplies, and once in its document's own, as the reader reads it (amended on 2026-10-23); a full + index, once a document. The filters after one a guard stopped, and those of data `MaxStreamLength` cut, are given data + the reader cut: that it runs out — a lost tail, a missing checksum, a digit without its pair, a row the predictor + cannot finish — is not reported, and a fault in what was read is. The code `filter.limit-exceeded`, added by T31 and + never released, becomes `limit.decoded-stream`. - **What a guard leaves out of the index is not damage** (amended on 2026-10-14). The objects only a section past `MaxXRefSectionCount`, rows past `MaxXRefSectionLength`, `MaxStreamLength` or `MaxDecodedStreamLength`, or numbers past `MaxIndexEntries` index read as null, and the validator treats them as unknown rather than missing, until the property is raised: the index is not rebuilt for them, which turned a sound file into an `xref.rebuilt` error (ADR 45) and could lose what the chain had read. A classic table cut by its length keeps its trailer. The chain ends at a cut — a table's, a stream's rows, a section or a trailer cut wherever the cut falls, a cross-reference stream - whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` leads to an object `MaxObjectLength` cut (amended on - 2026-10-23) —, since an older section, or its trailer, would stand for what the cut left out; and a section a guard cut is where it was named, not looked for - near there. A rebuild that damage forces on such a chain keeps the entries the chain read; one the chain cannot do - without — a guard that leaves it no section — runs as before. + whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` leads to an object `MaxObjectLength` cut, or whose + `/Length` does where only it could say how far the data runs, a `/Prev` or an `/XRefStm` that does (amended on + 2026-10-23) —, since an older section, or its trailer, would stand for what the cut left out; and a section a guard + cut is where it was named, not looked for near there. A rebuild that damage forces on such a chain keeps the entries + the chain read; one the chain cannot do without — a guard that leaves it no section — runs as before. - **Or it throws.** `PdfReaderOptions.ThrowOnLimit`, false by default, turns every guard reached into a `PdfLimitExceededException`, a `PdfException` carrying the code and the name of the property: from `PdfDocument.Open` when it is reached while opening, from `GetObject`, `GetRawBytes` or `Decode` when it is reached later. diff --git a/docs/architecture.md b/docs/architecture.md index ce0a94cb..7e32c5bc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -98,12 +98,13 @@ separate types, separate severity scales, and vocabularies that never share a co (section 5), and decodes under the limits of the document it came from; a stream built in memory decodes under the defaults. A stream copied from one document to another travels **encoded**, with no decompress/recompress cycle. Its declared length is checked when it is parsed, against the `endstream` that - must follow it after white space and comments, however many: in the window the object is parsed through, or - in the file past it, read on as far as the white space and comments run. A length nothing confirms, or a - `/Length` that gives none, goes to one search for the first `endstream` after the data's start ([#174]): in - the bytes the parser holds, then in the file — once per stream, no further than the nearer of the next object - the file's index places, as the file wrote it (as the reader rebuilt it when the document opened, when the - file wrote none), and the first object header the file's bytes hold, or the end of the file, and the + must follow it after white space and comments, however many — in the window the object is parsed through, or + in the file past it, read on as far as they run —, a comment counting only once white space comes before it: a + `%` right where the length ends is the data's, and leaves the length unconfirmed. A length nothing confirms, or + a `/Length` that gives none, goes to one search for the first `endstream` after the data's start ([#174]): in + the bytes the parser holds, then forward in the file — once per stream, no further than the nearer of the next + object the file's index places, as the file wrote it (as the reader rebuilt it when the document opened, when + the file wrote none), and the first object header the file's bytes hold, or the end of the file, and the searches of a document reading a few times the file at most. A stream's data never counts toward its object's length: no window grows to find its `endstream`. Neither bound changes as objects are read: what a search keeps depends on the file's bytes and its index as written, and in the 401 corpus documents measured @@ -111,18 +112,29 @@ separate types, separate severity scales, and vocabularies that never share a co crafted file can still make it depend on the number a stream is first read under, on the rebuild's scan for trailers, or, once the searches of a document have read four times the file, on which streams were searched first ([#138], M23). With no `endstream` before the stop, the object's `endobj` ends the data where it - follows the declared length, or where it ends the object before the stop; without one, a declared length - that ends before the stop is kept, and the data otherwise runs to the stop — to the end of the file, for a - stream the file ends inside. Each is a diagnostic. + follows the declared length as an `endstream` would, or where it is the last one, a whole keyword, and only + white space and comments run from it to the stop. That last `endobj` is followed forward through the search's + own reads, which read every byte up to the stop already: no byte before the stop is read a second time, + backward, and a comment, which can be told only from the start of its line, is read as one. Without either, + a declared length that ends before the stop is kept, and the data otherwise runs to the stop — to the end of + the file, for a stream the file ends inside. Each is a diagnostic. What the search found is kept for the + stream; whether `endobj` follows the declared length is read again only when a rebuild changes that length. 5. **Guards** — every read sized by the file is bounded. A bound a valid file can exceed — a stream's data read at once and its decoded length, the filters it passes through, an object's length, a cross-reference section's length and their number, a trailer's length, the entries of the index — is a `PdfReaderLimits` option, on by default and reported under its own `limit.*` code when reached; a bound only an invalid file reaches is an internal constant (ADR 34). What a guard cut is the guard's, never the file's fault: a value that leads to an object a guard cut or left out of the index — a stream's `/Length`, `/Filter` or - `/DecodeParms`, a cross-reference stream's `/W` — is not judged as one the file gave: the stream's data is - found by its `endstream`, its chain of filters ends there with the data left encoded, and the chain of - cross-reference sections ends at that section. + `/DecodeParms`, a cross-reference stream's `/W`, a trailer's `/Prev` or `/XRefStm` — is not judged as one the + file gave. The stream's data is found by its `endstream`, an integer the cut left only a place to look for + one at, and a missing `endstream` is still the file's fault; its chain of filters ends there, with the data + left encoded; and the chain of cross-reference sections ends at that section, with nothing rebuilt and the + older sections and their trailers the guard's to hide. An object `MaxObjectLength` cut there is loaded once + the chain is read, which reaches the guard — thrown at opening under `ThrowOnLimit`. A cross-reference + stream's `/Length` the chain could not read, which leads to such an object, ends the chain so wherever only it + could say how far the data runs: with no `endstream` before the next object or the end of the file, the + section is the guard's, not malformed; where the rows spell one and no `endstream` confirms what the cut left + of the length, the rows past it are the guard's. ### 3.2 The write path diff --git a/docs/milestones/M02.md b/docs/milestones/M02.md index 66491628..793dbab6 100644 --- a/docs/milestones/M02.md +++ b/docs/milestones/M02.md @@ -101,22 +101,26 @@ stream, and the definition of it, that the chain's index places: an object a reb list it or writes it as a stream, reads as null with no finding yet ([#276]). Where a stream's data ends is settled once for each stream, by the same rules wherever the parser's window ends: a -declared length that an `endstream` follows, past white space and comments however many, is the stream's; any other, or -none, goes to one search for the first `endstream`, up to the nearer of the first object header and the next object the -index as written places, the object's `endobj` standing in for it when none lies before; and a stream's data never -counts toward `MaxObjectLength` ([#174]). A cross-reference stream whose `/Length` the chain cannot read is searched -alike past the window, with no header stopping it; where the section's own row for that `/Length` lies among rows that -spell `endstream`, the data is taken again to the length it gives, and data no `endstream` ends before the next object -the sections read so far place makes the section malformed, the file's fault and no guard's ([#220]). The next object's -header ends every array and dictionary an object, or a trailer, left open, and the object lacks its `endobj` ([#230]). +declared length that an `endstream` follows, past white space and comments however many, a comment only after white +space, is the stream's; any other, or none, goes to one search for the first `endstream`, up to the nearer of the first +object header and the next object the index as written places, the object's `endobj` standing in for it when none lies +before; and a stream's data never counts toward `MaxObjectLength` ([#174]). A cross-reference stream whose `/Length` the +chain cannot read is searched alike past the window, with no header stopping it; where the row the section's reading +keeps for that `/Length` lies among rows that spell `endstream`, the data is taken again to the length it gives, and +data no `endstream` ends before the next object the sections read so far place makes the section malformed, the file's +fault and no guard's, unless that `/Length` leads to an object a guard cut ([#220]). The next object's header ends every +array and dictionary an object, or a trailer, left open, and the object lacks its `endobj` ([#230]). A stream's `/Length`, `/Filter` or `/DecodeParms`, or a filter's parameter, that leads to an object a guard cut or left -out of the index is the guard's, never judged as a value the file gave: the length is not judged, and the data is found -by its `endstream`; the filters end there, the rest of them left undone, and an object stream so left is hidden as one a -guard cut, `xref.checked-in-part`'s rather than `xref.object-stream-broken`'s. A cross-reference stream whose `/W`, -`/Index`, `/Size`, `/Filter` or `/DecodeParms` leads to an object `MaxObjectLength` cuts while the chain is read ends the -chain as a section a guard cut, and nothing is rebuilt ([#224], [#263]). A stream's data is read each time it is asked -for and kept by nothing, at most `MaxStreamLength` of it at once, a guard: validating object streams whose data share a -stretch of the file holds no copy of it for each ([#170]). +out of the index is the guard's, never judged as a value the file gave: the length is not judged, the data is found by +its `endstream`, and one missing is still the file's fault; the filters end there, the rest of them left undone, +`HasImageFilter` seeing no image past it, and an object stream so left is hidden as one a guard cut, +`xref.checked-in-part`'s rather than `xref.object-stream-broken`'s. A cross-reference stream whose `/W`, `/Index`, +`/Size`, `/Filter` or `/DecodeParms` — or whose `/Length`, when no `endstream` ends its data — leads to an object +`MaxObjectLength` cuts while the chain is read ends the chain as a section a guard cut, as a `/Prev` or `/XRefStm` that +leads to one does, and nothing is rebuilt ([#224], [#263]). A stream's data is read each time it is asked for and kept +by nothing — validating object streams whose data share a stretch of the file holds no copy of it for each —, and no +more than `MaxStreamLength` of it is read at once, a guard reported once in the document's diagnostics and with each +decoding into diagnostics a caller passes ([#170]). ## Slices diff --git a/docs/threat-model.md b/docs/threat-model.md index b6aacfda..8db3d658 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -108,13 +108,16 @@ programs, image files, content streams, XML, encryption, signatures and the netw Reaching a guard keeps what fits and reports its code (`PdfLimitGuard.Reach`): once per position for an object written in the file, a section, a trailer or a stream's data, once for each member of each definition of an object stream — at that stream's data start, which its members share —, once per document for the index, and for a stream decoded past the bound once in its document's -diagnostics, each time into diagnostics the caller passes, counted each time either way ([#144]). What a guard +diagnostics, each time into diagnostics the caller passes, counted each time either way ([#144]); data `MaxStreamLength` +cut, reported once at its position in the document's diagnostics, reaches diagnostics the caller passes with each decoding +that meets the cut too ([#170]). What a guard leaves out of the index reads as null until it is raised, and the index is not rebuilt for it: a guard's cut is no damage. A value read through a reference to an object a guard cut or left out of the index, through a chain of references too, is the guard's as well, whatever the cut left of it — null, or a token it parted —, and the reader judges none of it as the file's. That holds for a stream's `/Length` ([#224]) and the values its filters read -([#263]), and for a cross-reference stream's `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` that -`MaxObjectLength` cut as the chain was read (*Indexing*); it adds no bound. When the document was opened with +([#263]), for a cross-reference stream's `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` that +`MaxObjectLength` cut as the chain was read, and its `/Length` when no `endstream` ends its data, and for a `/Prev` or +`/XRefStm` that leads to such an object (*Indexing*); it adds no bound. When the document was opened with `ThrowOnLimit`, reaching a guard throws `PdfLimitExceededException` instead. A stream read from a document decodes under that document's guards, however long after opening. `PdfReaderLimits` refuses a bound of zero or less and takes a length past `Array.MaxLength` as that length. Held by `ReaderLimitsTests` throughout; the @@ -245,7 +248,7 @@ Known gaps: | The lexer looping, or reading past its buffer | Every token but the end of input consumes a byte. Look-aheads check the length, and a string that never closes ends at the buffer's end, and is told from a closed one by its token's length | `PropertyTests.The_lexer_terminates_on_any_bytes_and_stays_inside_the_buffer`; `LexerTests.Never_loops_on_a_stray_delimiter`, `Recovers_from_an_unterminated_literal_string`; `ParserTests.Reports_nothing_of_a_construct_closed_at_the_end_of_the_data` | Property, over buffers of about 100 bytes | | Nesting that exhausts the stack | `PdfObjectParser.MaxDepth` (128) and a stack probe at each container. Past either, `SkipContainer` counts brackets without recursion, and `syntax.depth-exceeded` is reported | `ParserTests.Refuses_to_follow_nesting_deeper_than_its_limit`; `HostileInputTests.Survives_deeply_nested_containers_inside_an_object`, `A_file_nesting_objects_deeper_than_the_stack_allows_reads_them_as_null_without_crashing` | No | | A token or an object of any length, or one that never closes | The window grows ×8 from 8 KB up to `MaxObjectLength`, and the parser is told whether its window ends the data: only then is what it leaves open reported, once, `syntax.truncated-object` where the innermost construct opens, and a window that ends where the file does cuts nothing however long it is. Past the bound, the object is kept as far as it was read and `limit.object` is reported, beside what the parse met before the cut; nothing is reported of the token the cut reaches, nor of what follows once a look-ahead met it (ADR 34). An array or a dictionary an `endobj` finds open ends there, rather than take the objects after it, and one that lost its own `endobj` too ends at the next object's header (below). The parses of one object add up to about 1.3 times the last window | `HostileInputTests.Survives_a_string_longer_than_the_largest_window`, `A_string_that_never_closes_takes_the_end_of_the_file_and_is_reported_once_where_it_opens`; `ReaderLimitsTests.An_object_is_read_no_further_than_its_bound`, `An_object_cut_by_its_bound_is_not_reported_as_damage`, `An_object_its_bound_cuts_anywhere_in_its_last_tokens_reports_the_bound_alone`, `What_an_object_its_bound_cuts_met_before_the_cut_is_still_reported`, `A_fault_after_a_stream_whose_length_the_file_cannot_hold_is_still_reported_beside_the_bound`, `An_object_its_bound_cuts_after_more_faults_than_the_diagnostics_hold_still_reports_the_bound`, `A_number_its_bound_cuts_is_no_number_out_of_range`, `An_object_as_long_as_its_bound_that_ends_the_file_is_read_whole`; `WindowEdgeTests.An_anomaly_inside_an_object_longer_than_the_window_is_reported_once`; `PropertyTests.What_a_window_reports_of_its_syntax_the_whole_data_reports_too_in_the_same_order`, `A_buffer_that_ends_at_a_window_s_edge_reports_nothing_of_the_end_of_the_data`; `SyntaxFaultTests.A_construct_the_file_ends_inside_is_reported_where_it_opens`, `A_container_an_endobj_ends_takes_nothing_of_the_objects_after_it`, `A_fault_of_an_object_read_through_a_window_the_file_goes_on_past_is_reported`, `A_rebuild_reports_no_fault_where_its_window_cuts_a_trailer`, `A_trailer_a_source_cut_short_ends_inside_is_reported_as_the_end_of_the_data`; `FileRuleTests.A_trailer_whose_closing_brackets_end_its_table_s_window_is_judged_by_what_they_close` | Growth yes, the bound no; the window's edge by property | -| An object that lost both its closing delimiter and its `endobj` taking in the next object, up to that object's `endobj`, its own missing `endobj` unreported; a trailer that lost its `>>` taking the next update's entries | Inside an array or a dictionary, where an element, a key or a value is expected, the next object's header ends every container left open: `N G obj`, digits alone, `N` in 1..`int.MaxValue` and `G` in 0..65,535, white space alone between them, and `obj` a whole keyword the window's edge does not reach — the header the search for a lost `endstream` stops at (`HeaderText`). It is reported once, at the innermost container, as `syntax.truncated-object`, "An object header ended the object inside …", and the object lacks its `endobj`. Containers `SkipContainer` passes over past `MaxDepth` end there too, and a trailer is read so as well. An object stream's member, which no header follows, and a value alone, which is no container, do not apply it: in `4 0 obj 5 0 obj`, object 4 holds the integer 5. Its cost, accepted: a closed container holding text that reads as a header ends there, and its `endobj` is judged missing; none of the 344,361 objects of the corpus holds one ([#230]) | `SyntaxFaultTests.An_object_that_lost_its_closing_delimiter_and_its_endobj_ends_at_the_next_object_s_header`, `The_next_object_s_header_ends_every_container_left_open_and_is_reported_at_the_innermost`, `Text_that_is_no_object_header_ends_no_container`, `A_header_inside_a_closed_container_ends_it_there_and_its_endobj_is_judged_missing`, `An_object_stream_member_holding_a_header_reads_it_as_tokens`, `Nesting_deeper_than_the_reader_follows_ends_at_the_next_object_s_header_too`, `A_trailer_that_lost_its_closing_delimiter_takes_nothing_of_the_update_after_it`; `ParserTests.Ends_the_containers_an_object_left_open_at_the_next_object_s_header_and_leaves_it_to_be_read`, `Takes_an_object_header_a_window_s_edge_reaches_for_none_until_a_larger_window_shows_obj_whole`, `A_header_where_a_key_or_a_value_is_expected_ends_the_dictionary`, `A_value_alone_that_is_a_header_s_number_is_an_integer`; `WindowEdgeTests.An_object_that_lost_its_end_ends_at_the_next_header_wherever_the_window_edge_cuts_it` | Plausibly | +| An object that lost both its closing delimiter and its `endobj` taking in the next object, up to that object's `endobj`, its own missing `endobj` unreported; a trailer that lost its `>>` taking the next update's entries; a header the window's edge cut short reported as the file's stray keys | Inside an array or a dictionary, where an element, a key or a value is expected, the next object's header ends every container left open: `N G obj`, digits alone, `N` in 1..`int.MaxValue` and `G` in 0..65,535, white space alone between them, and `obj` a whole keyword the window's edge does not reach — the header the search for a lost `endstream` stops at (`HeaderText`). A header the edge cuts short, `6 0` or `6 0 ob`, is the edge's, as a reference it cuts is: where no larger window follows — at `MaxObjectLength`, in the rebuild's trailer scan —, its numbers are neither reported as keys that are not names nor taken for the value of the key before them. It is reported once, at the innermost container, as `syntax.truncated-object`, "An object header ended the object inside …", or, in a trailer or a value read alone, where no object is open, "An object header came inside …", and the object lacks its `endobj`. Containers `SkipContainer` passes over past `MaxDepth` end there too, and a trailer is read so as well. An object stream's member, which no header follows, and a value alone, which is no container, do not apply it: in `4 0 obj 5 0 obj`, object 4 holds the integer 5. Its cost, accepted: a closed container holding text that reads as a header ends there, and its `endobj` is judged missing; none of the 344,361 objects of the corpus holds one ([#230]) | `SyntaxFaultTests.An_object_that_lost_its_closing_delimiter_and_its_endobj_ends_at_the_next_object_s_header`, `The_next_object_s_header_ends_every_container_left_open_and_is_reported_at_the_innermost`, `Text_that_is_no_object_header_ends_no_container`, `A_header_inside_a_closed_container_ends_it_there_and_its_endobj_is_judged_missing`, `An_object_stream_member_holding_a_header_reads_it_as_tokens`, `Nesting_deeper_than_the_reader_follows_ends_at_the_next_object_s_header_too`, `A_trailer_that_lost_its_closing_delimiter_takes_nothing_of_the_update_after_it`, `A_header_the_guard_s_cut_leaves_undecided_is_the_guard_s`; `ParserTests.Ends_the_containers_an_object_left_open_at_the_next_object_s_header_and_leaves_it_to_be_read`, `Takes_an_object_header_a_window_s_edge_reaches_for_none_until_a_larger_window_shows_obj_whole`, `A_header_where_a_key_or_a_value_is_expected_ends_the_dictionary`, `A_value_alone_that_is_a_header_s_number_is_an_integer`, `Tokens_a_window_s_edge_leaves_undecided_as_a_header_are_no_stray_keys`; `WindowEdgeTests.An_object_that_lost_its_end_ends_at_the_next_header_wherever_the_window_edge_cuts_it` | Plausibly | | Look-ahead that makes parsing superlinear | Look-ahead is a constant number of tokens, so each token is lexed three times at most | `ParserTests.Distinguishes_a_reference_from_two_integers`, `Backtracks_when_two_integers_are_not_followed_by_R` | Property | | A number past a `long` or a `double` | The number parser never wraps: an integer past a `long` reads as the real nearest to it, and a number past a double's range as an infinity, which the object parser reads as null under `syntax.number-out-of-range`, quoting the number only in a report the diagnostics keep. A real reads as the double nearest to its decimal. The shape — signs, then digits around at most one period — is checked before the framework's parse, which would read `NaN` and `Infinity`; each digit is read twice at most, and nothing is allocated | `LexerTests.Reads_an_integer_too_large_for_a_long_as_a_real`, `Reads_a_number_past_the_largest_double_as_an_infinity_its_reader_can_tell`, `Parsing_a_number_allocates_nothing_however_long_it_is`; `PropertyTests.The_integer_parser_reads_every_long_and_never_wraps_past_one`, `The_real_parser_reads_the_double_nearest_to_the_decimal`; `ParserTests.Reads_a_number_past_the_largest_real_as_null_and_reports_it`, `Reports_no_number_its_double_holds_however_it_rounds`; `HostileInputTests.Reads_a_number_fifteen_million_digits_long_in_time`, `Quotes_no_number_beyond_what_a_real_can_hold_once_the_diagnostics_are_full` | Property | | An object number or a generation overflowing its cast | The parser builds a header only from a number in 1..`int.MaxValue`, and a reference only from one in 0..`int.MaxValue` — object 0 heads the free list and reads as null —, each with a generation in 0..65,535 (`PdfObjectId.MaxNumber` and `MaxGeneration`, with their reasons), checked before the cast. The index and object streams check theirs the same way, before narrowing: see *Indexing* and *Resolving objects and object streams* | `ParserTests.Refuses_an_object_header_whose_number_or_generation_is_out_of_range`, `Makes_no_reference_of_numbers_an_object_identifier_cannot_hold`; `HostileInputTests.Serves_no_object_under_a_number_past_a_long_that_once_wrapped_to_its_number` | No | @@ -274,13 +277,13 @@ Known gaps: |---|---|---|---| | A `/Prev` chain that loops | The offsets the file writes are kept, and a repeat ends the chain with `xref.chain-cycle`, placed at an offset in the file; two offsets past what a `long` holds are two, not one | `DocumentReaderTests.Stops_when_the_chain_of_sections_loops`; `CrossReferenceChainTests.Reports_a_loop_in_the_chain_at_the_section_it_loops_back_to_whatever_precedes_the_header`, `Places_a_loop_back_to_a_section_named_outside_the_file_at_what_named_it`; `CrossReferenceRuleTests.A_prev_that_names_a_section_already_read_is_a_loop_and_an_error`; `HeaderOffsetTests.A_chain_that_names_the_same_offset_outside_the_file_twice_loops`, `Two_offsets_past_what_a_long_holds_name_two_sections_not_one_loop`; `CrossReferenceRuleTests.A_loop_through_an_offset_the_chain_only_named_says_no_section_was_read_there` | Yes | | A chain without end | `MaxXRefSectionCount` counts the `/Prev` links; a cut where the chain goes on outside the file is placed at the section that names it, the offset in the message | `ReaderLimitsTests.Reaching_a_guard_keeps_what_fits_and_names_the_property_that_lifts_it`; `CrossReferenceChainTests.Reads_an_object_only_a_section_past_the_section_count_indexes_as_null_until_the_guard_is_lifted`; `HeaderOffsetTests.A_chain_cut_where_it_goes_on_past_what_a_long_holds_is_placed_at_the_section_naming_it`, `A_chain_cut_outside_the_file_names_where_it_goes_on_as_the_file_writes_it_the_header_added` | No | -| A section offset outside the file; `/Prev` or `/XRefStm` given as a reference | Offsets are checked against the file, before the header's offset is added. A `/Prev` or an `/XRefStm` written as a reference is read where a section already read places its object, without loading it, and a real with no fractional part as its integer; anything else names no section | `CrossReferenceChainTests.Reports_a_section_that_prev_names_past_the_end_of_the_file_once_at_the_section_naming_it`, `Reports_a_prev_that_is_not_an_offset_and_finds_what_it_named_when_asked`; `CrossReferenceRuleTests.A_prev_written_as_a_reference_or_a_real_is_followed_and_reported`, `A_hybrid_file_whose_xrefstm_is_a_reference_or_a_real_is_read_whole` | Yes | +| A section offset outside the file; `/Prev` or `/XRefStm` given as a reference | Offsets are checked against the file, before the header's offset is added. A `/Prev` or an `/XRefStm` written as a reference is read where a section already read places its object, without loading it, and a real with no fractional part as its integer; anything else names no section, but for one that leads to an object a guard cut or hid, which ends the chain for the guard (below) | `CrossReferenceChainTests.Reports_a_section_that_prev_names_past_the_end_of_the_file_once_at_the_section_naming_it`, `Reports_a_prev_that_is_not_an_offset_and_finds_what_it_named_when_asked`; `CrossReferenceRuleTests.A_prev_written_as_a_reference_or_a_real_is_followed_and_reported`, `A_hybrid_file_whose_xrefstm_is_a_reference_or_a_real_is_read_whole` | Yes | | A missing section tried at many places | Candidates within 512 bytes either side, nearest first, and 32 attempts at most per document (`MaxRelocationCandidates`); a section a guard cut is where it was named, and is tried nowhere else | `CrossReferenceChainTests.Reads_a_section_that_prev_names_a_few_bytes_off`, `Sections_named_a_few_bytes_off_are_looked_for_at_thirty_two_places_a_document_at_most`; `HostileInputTests.Opening_a_chain_of_hybrid_sections_whose_streams_never_close_reads_a_bounded_amount` | Yes | | A classic table of any length | Its window grows ×4 from 64 KB up to `MaxXRefSectionLength`, and the rows read are kept. A table the guard cuts, or whose rows run past a full index, keeps its trailer, found past the rows left unread by a search a megabyte at a time (`ReadTrailerPastRows`): comments are skipped, and anything else no row holds is the section's fault, reported, the search going on to the `trailer` keyword, so that its `/Encrypt` still refuses the document. The search starts at the start of the line the cut fell in, outside any comment and past the table's own keyword; it reads each megabyte once, every byte of it looked at before the next is read, however many comments or stray bytes it holds, and is not made for a place a section may have been moved to, a guess. The chain ends there, since an older section's rows would stand for the objects the unread rows define. What those rows place reads as null until the guard is raised | `ReaderLimitsTests.A_table_whose_keywords_the_bound_cuts_reports_the_bound`, `A_table_its_length_bound_cuts_keeps_its_trailer_and_the_encryption_it_names`, `A_table_its_length_bound_cuts_ends_the_chain_though_its_trailer_names_an_older_section`, `What_ends_the_rows_a_length_bound_cut_is_judged_as_it_is_where_none_did`, `A_table_a_length_bound_cuts_that_runs_to_the_end_of_the_file_has_no_trailer`, `A_table_its_length_bound_cuts_keeps_the_encryption_its_trailer_names_past_what_no_row_holds`, `A_table_cut_before_its_keyword_ends_keeps_its_trailer`, `Rows_past_a_full_index_are_not_read_and_the_trailer_past_them_is`, `A_cut_in_a_comment_among_the_rows_searches_for_the_trailer_from_the_comment_s_start`, `The_trailer_past_rows_left_unread_is_found_wherever_the_search_s_window_ends`, `A_trailer_past_rows_left_unread_that_its_own_bound_cuts_keeps_what_precedes_the_cut`; `DocumentReaderTests.The_search_for_the_trailer_past_rows_left_unread_reads_the_file_once`; `CrossReferenceChainTests.Reads_an_object_a_length_bound_cut_out_of_its_table_as_null_until_the_guard_is_lifted`, `A_section_found_near_where_prev_names_it_is_not_searched_past_the_rows_a_guard_left_unread`; `WindowEdgeTests.A_classic_table_reads_whole_wherever_its_window_edge_falls_near_its_trailer` | No: seeds stay under 120 KB | | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | -| A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one` | Yes | -| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms`, or an element or a parameter of one: the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263]). A `/Length` is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length holds the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …". Rows that spell an `endstream` are told from the data's end by the section's own row for the `/Length`, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. A row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | +| A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s` | Yes | +| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length holds the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places, the integer read before the cut is only a place to look for `endstream` at: the data is taken again where one confirms it, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, and nothing is charged to the file ([#224]). A row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data `MaxStreamLength` cut is the guard's, and not checked against it ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | @@ -345,17 +348,17 @@ Known gaps: | Threat | Defense | Held by | Fuzzed | |---|---|---|---| | A `/Length` that is negative, past `int.MaxValue`, not an integer or unresolvable | Only 0..`int.MaxValue` is taken as a length. Anything else sends the reader to the one search for the stream's first `endstream` (below) | `StreamLengthTests.A_length_that_cannot_be_taken_is_reported_as_the_file_wrote_it` | Partly | -| A `/Length` naming an object a guard cut or left out of the index, judged by what the cut left of it: null, reported as no length, or the `1` it parted of `15`, reported as a length the data contradicts | Keyed on the object the reference leads to, through a chain of references too, never on the value read (`StreamLengthForm.ObjectCut`, `ObjectUnindexed`): an integer read before the cut is only a place to look for `endstream` at, never kept as a length, and the data is otherwise found by its `endstream`. Nothing is said of the length, whose fault is the guard's, reported where it was reached; a missing `endstream` is still the file's, and reported: "The stream's /Length names object N G, which one of the reader's limits cut, and no endstream follows its data …". A cross-reference stream's `/Length` so cut is left alike once the chain is read ([#224]) | `GuardCutValueTests.A_length_a_guard_cut_is_not_judged_and_the_data_is_found_by_its_endstream`, `A_length_a_guard_cut_leaves_a_missing_endstream_the_file_s_fault`, `A_length_an_object_stream_s_guard_left_unread_is_not_judged`, `A_length_whose_reference_leads_to_an_object_a_guard_left_out_of_the_index_is_not_judged`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read` | No | +| A `/Length` naming an object a guard cut or left out of the index, judged by what the cut left of it: null, reported as no length, or the `1` it parted of `15`, reported as a length the data contradicts; or the stream's missing `endstream` left unreported along with the length | Keyed on the object the reference leads to, through a chain of references too, never on the value read (`StreamLengthForm.ObjectCut`, `ObjectUnindexed`): an integer read before the cut is only a place to look for `endstream` at, never kept as a length, and the data is otherwise found by its `endstream`. Nothing is said of the length, whose fault is the guard's, reported where it was reached; a missing `endstream` is still the file's, and reported, naming the object at the end of the chain of references, the one the guard cut or left out: "The stream's /Length names object N G, which one of the reader's limits cut, and no endstream follows its data …", or "…, which one of the reader's limits left out of the index, and no endstream follows its data …". A cross-reference stream's `/Length` so cut is left alike once the chain is read, and makes its section the guard's when no `endstream` ends its data, as *Indexing* says ([#224]) | `GuardCutValueTests.A_length_a_guard_cut_is_not_judged_and_the_data_is_found_by_its_endstream`, `A_length_a_guard_cut_leaves_a_missing_endstream_the_file_s_fault`, `A_length_an_object_stream_s_guard_left_unread_is_not_judged`, `A_length_whose_reference_leads_to_an_object_a_guard_left_out_of_the_index_is_not_judged`, `A_missing_endstream_names_the_object_a_guard_cut_at_the_end_of_the_length_s_chain_and_keeps_no_length`, `A_missing_endstream_is_the_file_s_fault_when_a_guard_left_the_length_out_of_the_index`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read` | No | | An indirect `/Length` leading back to its stream, or chained | The loading set and `MaxNestedLoads`, as above | `HostileInputTests.Survives_an_object_whose_length_refers_to_itself`, `Survives_a_chain_of_lengths_longer_than_the_stack_is_deep` | No | | A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found: no window is grown to find an `endstream`, the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_data_start_a_limit_cuts_records_nothing_of_its_length`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows` | No | -| Confirming a length by reading past the buffer or the file, or through white space without end | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. The bytes the parser holds decide when they can. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none` | Property | -| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space between it and the stop, ends the data, its end-of-line left out, `stream.truncated`. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more` | Plausibly | +| Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none` | Property | +| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more` | Plausibly | | Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, and by white space after a declared length that runs past a first read, about twice the file at most; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | -| The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go` | — | -| Encoded data of any length: a `/Length` of a few digits claiming gigabytes, one past what an array holds, which would exhaust memory at each read, or data decoding hands back as it is, which `MaxDecodedStreamLength` does not bound | `MaxStreamLength` (*The guards*): what is read of one stream's data at once stops at it, what lies past it is cut by a guard, reported once for the stream as `limit.stream-length`, and the filters decode what was read as data the reader cut ([#170]) | `ReaderLimitsTests.A_stream_cut_at_MaxStreamLength_decodes_what_lies_within_it_as_data_the_reader_cut_and_is_reported_once`, `Data_decoding_hands_back_as_it_is_is_bounded_by_MaxStreamLength_and_not_by_MaxDecodedStreamLength`, `A_length_past_what_an_array_holds_ends_at_the_guard_and_never_in_an_exhausted_memory` | No | -| A stream's raw data held for as long as the stream is, and streams whose data share a stretch of the file each holding a copy of it: two hundred sharing a megabyte, 200 MB | Nothing keeps a stream's encoded data (`FileStreamData`): it is read each time it is asked for, by `GetRawBytes` or a decoding. A source in memory lends it where it lies, with no copy, and any other reads it into an array of its length. A stream keeps its place and length, not its bytes ([#170]) | `DocumentReaderTests.A_stream_s_data_is_read_from_the_file_each_time_it_is_asked_for_and_kept_by_nothing`; `HostileInputMemoryTests.Streams_whose_data_share_a_stretch_of_the_file_hold_none_of_it_once_read`, `Validating_object_streams_whose_data_share_a_stretch_of_the_file_holds_no_copy_of_their_data`; `HostileInputTests.Reading_streams_whose_data_share_a_stretch_of_a_file_held_in_memory_allocates_none_of_their_data` | No | +| The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | +| Encoded data of any length: a `/Length` of a few digits claiming gigabytes, one past what an array holds, which would exhaust memory at each read, or data decoding hands back as it is, which `MaxDecodedStreamLength` does not bound | `MaxStreamLength` (*The guards*): what is read of one stream's data at once stops at it, what lies past it is cut by a guard and reported as `limit.stream-length` — once for the stream in its document's diagnostics, and with each decoding that meets the cut in diagnostics a caller passes to `Decode`, as `limit.decoded-stream` is —, and the filters decode what was read as data the reader cut ([#170]) | `ReaderLimitsTests.A_stream_cut_at_MaxStreamLength_decodes_what_lies_within_it_as_data_the_reader_cut_and_is_reported_once`, `Data_decoding_hands_back_as_it_is_is_bounded_by_MaxStreamLength_and_not_by_MaxDecodedStreamLength`, `A_length_past_what_an_array_holds_ends_at_the_guard_and_never_in_an_exhausted_memory`, `Decoding_into_diagnostics_of_one_s_own_reports_the_cut_max_stream_length_made_each_time` | No | +| A stream's raw data held for as long as the stream is, and streams whose data share a stretch of the file each holding a copy of it: two hundred sharing a megabyte, 200 MB | Nothing keeps a stream's encoded data (`FileStreamData`): it is read each time it is asked for, by `GetRawBytes` or a decoding. A source in memory lends it where it lies, with no copy — a view of that memory, which keeps it alive after the document is disposed for as long as it is held, and changes as it does —, and any other reads it into an array of its length. A stream keeps its place and length, not its bytes ([#170]) | `DocumentReaderTests.A_stream_s_data_is_read_from_the_file_each_time_it_is_asked_for_and_kept_by_nothing`; `HostileInputMemoryTests.Streams_whose_data_share_a_stretch_of_the_file_hold_none_of_it_once_read`, `Validating_object_streams_whose_data_share_a_stretch_of_the_file_holds_no_copy_of_their_data`; `HostileInputTests.Reading_streams_whose_data_share_a_stretch_of_a_file_held_in_memory_allocates_none_of_their_data` | No | | A stream decoded long after opening escaping its document's guards | The stream's data carries its document's guard and diagnostics | `ReaderLimitsTests.A_stream_decoded_without_diagnostics_reports_the_bound_to_its_document` | Yes | Known gaps: @@ -383,7 +386,7 @@ its zlib patched is keeping part of the reader patched. | A `/Filter` array of any length, each filter decoding to the bound | The filters before the last share one `MaxDecodedStreamLength` for what they pass along together, and the last has its own: a chain holds and fills two bounds at most, whatever its length. A `/Crypt` filter, which passes its data along as it is, is neither charged nor the last. A filter that decodes nothing ends the chain, so that a faulty Flate stream is read again once, not once an entry ([#166]) | `FilterTests.Shares_one_bound_among_the_filters_before_the_last`, `Bounds_a_filter_in_the_middle_of_a_chain_as_well_as_the_last_and_reports_the_first_that_stopped`, `Ends_a_chain_at_a_flate_step_that_decodes_nothing_and_returns_what_the_filters_before_it_decoded`, `A_crypt_filter_is_neither_charged_to_the_bound_the_filters_share_nor_the_last`; `HostileInputTests.Opens_a_cross_reference_stream_whose_run_length_filters_multiply_its_data_at_a_cost_its_number_of_filters_leaves_alone`, `Decodes_a_long_chain_of_sound_flate_layers_within_twice_the_bound`, `Decodes_data_no_flate_reading_can_decode_once_however_many_flate_filters_name_it`; `FilterDamageTests.A_chain_of_filters_decodes_to_its_data_or_to_the_start_of_it_with_the_guard_alone_reported` | Property | | A `/Filter` array of millions of filters, each costing decoding whatever its data | ISO 32000 sets no limit, and each filter costs a few microseconds, LZW a table of 42 KB, whatever its data; a filter whose data decodes to itself — `/RunLengthDecode` over `FF FF`, an identity `/Crypt` — never runs out of it. `MaxFilterChainLength` (64) bounds the filters a stream's data passes through: decoding stops before the first past it, and the elements past it are not read, by decoding, `HasImageFilter` or the cross-reference stream's check alike. Data a filter decodes to nothing ends decoding. What a chain keeps of what it read is what the file did not write plainly, within the entries the guard lets run: nothing for a name the array holds, the name an element written as a reference leads to, and only the values of parameters that differ from none at all; an array of forty million names, which an object stream's member holds in 350 KB of file, ran a decode out of memory when the chain kept 32 bytes for each | `HostileInputTests.Decodes_a_stream_naming_a_million_filters_at_the_cost_of_as_many_as_the_guard_lets_run`, `Decodes_a_stream_naming_a_million_filters_without_keeping_anything_for_the_names_its_array_holds`, `Decodes_nothing_further_once_a_filter_has_decoded_a_stream_to_nothing`; `FilterDamageTests.Gives_a_dictionary_of_parameters_beside_a_chain_to_its_flate_and_its_lzw_steps_alike`, `Reads_a_parameter_past_an_int_as_the_int_nearest_it_rather_than_its_low_bits`; `FilterTests.Reads_nothing_past_the_most_filters_the_guard_lets_a_stream_pass_through`, `Says_a_stream_is_an_image_only_when_its_image_filter_stands_within_the_guard`, `Reads_the_filters_a_chain_names_through_references_in_their_order_crypt_filters_among_them`; `CrossReferenceRuleTests.A_cross_reference_stream_s_filters_past_the_guard_are_not_followed`; `ReaderLimitsTests.Reaching_a_guard_keeps_what_fits_and_names_the_property_that_lifts_it`, `A_stream_the_reader_reads_whose_filters_pass_the_guard_is_cut_before_its_data` | No | | A guard's cut reported as damage by the filters after it | Every filter after one the bound stopped, or after `MaxStreamLength` cut the data, is given data the reader cut: no lost tail or missing checksum, no rows longer than the data; a hexadecimal digit or an ASCII85 group the cut parted is dropped, not completed, and no end marker is missing from it. A fault, or a checksum that disagrees, in what was read stays the file's ([#199], [#141]) | `FilterDamageTests.Says_nothing_of_the_end_of_data_a_filter_before_it_stopped_at_the_bound`, `Reports_a_fault_in_the_data_a_filter_kept_though_a_filter_before_it_stopped_at_the_bound`, `Drops_the_unit_a_guard_of_the_reader_parted_rather_than_complete_it`, `Completes_the_unit_its_end_marker_closes_however_the_reader_cut_the_data_after_it`, `Undoes_a_predictor_over_the_rows_of_a_bounded_lzw_output_and_the_part_of_the_last_it_holds`; `FilterTests.Undoes_a_predictor_over_the_rows_of_a_bounded_output_and_the_part_of_the_last_it_holds`, `Leaves_the_partial_last_row_of_data_the_reader_did_not_cut_as_the_predictor_left_it` | Property | -| A `/Filter`, a `/DecodeParms` or a step's parameter that leads to an object a guard cut or left out of the index, read as no filter, or as the `/Flate` a cut parted of `/FlateDecode`: an object stream's header read as stray bytes, its catalog looked for elsewhere, a sound file under a lowered limit judged broken | The chain of filters ends at such a value, through a chain of references too, as at the element past `MaxFilterChainLength` (`FilterChainEnd.Guard`): the steps before it run, the data is left encoded by the rest (`leftEncoded`), and nothing more is reported, the guard having been reported where it was reached. An object stream so left is hidden and its members are the guard's: a catalog among them is taken as read, and the validator reports `xref.checked-in-part`, not `xref.object-stream-broken`. Decoding says whether the stream's own data is cut — by `MaxStreamLength`, or by a filter stopped at `MaxDecodedStreamLength` —, which is what the readers of rows and members ask: a guard that loading a filter's value reaches is that value's, and hides none of the stream's own faults ([#263]) | `GuardCutValueTests.An_object_stream_whose_filter_values_a_guard_cut_is_hidden_and_its_members_taken_as_read`, `A_content_stream_whose_filter_a_guard_cut_comes_back_as_the_filters_before_it_decode_it`, `A_guard_a_filter_value_reaches_as_it_loads_is_not_the_stream_s_own_cut` | No | +| A `/Filter`, a `/DecodeParms` or a step's parameter that leads to an object a guard cut or left out of the index, read as no filter, or as the `/Flate` a cut parted of `/FlateDecode`: an object stream's header read as stray bytes, its catalog looked for elsewhere, a sound file under a lowered limit judged broken; a stream said to be an image whose data such a value leaves Flate-encoded | The chain of filters ends at such a value, through a chain of references too, as at the element past `MaxFilterChainLength` (`FilterChainEnd.Guard`): the steps before it run, the data is left encoded by the rest (`leftEncoded`), and nothing more is reported, the guard having been reported where it was reached. An object stream so left is hidden and its members are the guard's: a catalog among them is taken as read, and the validator reports `xref.checked-in-part`, not `xref.object-stream-broken`. Decoding says whether the stream's own data is cut — by `MaxStreamLength`, or by a filter stopped at `MaxDecodedStreamLength` —, which is what the readers of rows and members ask: a guard that loading a filter's value reaches is that value's, and hides none of the stream's own faults ([#263]). `HasImageFilter` reads what decoding reads before the image filter — the parameters of a Flate or LZW step before it, which it may load, decoding the object stream that holds one, as decoding would — and says no where such a value ends the chain first: an extractor that trusts it is not handed encoded data as the image | `GuardCutValueTests.An_object_stream_whose_filter_values_a_guard_cut_is_hidden_and_its_members_taken_as_read`, `A_content_stream_whose_filter_a_guard_cut_comes_back_as_the_filters_before_it_decode_it`, `A_guard_a_filter_value_reaches_as_it_loads_is_not_the_stream_s_own_cut`, `A_parameter_a_guard_cut_before_an_image_filter_leaves_no_encoded_image`; `FilterTests.Says_a_stream_is_an_image_only_when_decoding_it_stops_at_an_image_filter` | No | | An output buffer sized from the file | The first buffer is an estimate from the input, capped at the bound; it doubles and never grows past it | `FilterTests.A_filter_s_first_buffer_is_sized_by_its_bound_not_by_its_input`, `A_bounded_output_never_grows_past_its_bound_and_hands_back_a_full_buffer_as_it_is` | Yes | | Predictor parameters that overflow or size a row before a check | The row length is computed in 64 bits and compared with the data before anything is allocated | `FilterTests.Leaves_the_data_as_decoded_when_the_predictor_describes_rows_it_cannot_hold` | Seeds only | | An LZW code past the table | A fixed table of 4,096 codes. A code not yet defined stops decoding, keeps what came before and reports `filter.failed` | `FilterDamageTests.Decodes_an_lzw_code_up_to_the_next_one_to_define_and_stops_at_any_past_it`; `FilterTests.Stops_an_lzw_stream_at_a_code_it_has_not_defined_and_says_so` | Two seeds | diff --git a/docs/website/docs/concepts/diagnostics.md b/docs/website/docs/concepts/diagnostics.md index b982649c..7a31c890 100644 --- a/docs/website/docs/concepts/diagnostics.md +++ b/docs/website/docs/concepts/diagnostics.md @@ -29,13 +29,17 @@ through a larger one when it runs past the edge. What the smaller window saw the without its end, is dropped with that attempt, so the report says what the object holds, not where a window happened to end. When `MaxObjectLength` allows no larger window, the object is kept as far as it was read and the guard is reported: what the parse met before the edge — a stray token, a key that is not a name — is reported beside -it, and nothing of the token the edge cut, nor of what that token leaves open, is. A stream's data is no part of what -a window must hold: where it ends is found in the file, whatever window its dictionary was read through, and it never -counts toward `MaxObjectLength`. +it, and nothing of the token the edge cut, nor of what that token leaves open, nor of the next object's header the edge +cut short inside an array or a dictionary — `6 0` or `6 0 ob`, whose number and generation would otherwise read as keys +that are not names, or the number as a value —, is. A stream's data is no part of what a window must hold: where it +ends is found in the file, whatever window its dictionary was read through, and it never counts toward +`MaxObjectLength`. A stream's declared length is confirmed by the `endstream` that must follow it, after any white space and comments, -however many: past the window, the reader asks the file for the bytes after the declared length rather than reading -the data, reading on as far as white space and comments run, and they tell it whether `endobj` follows too. Any other +however many — a comment only once white space comes before it: ISO 32000-1 (7.2.3) counts a comment as white space +outside a stream's data, and a `%` right where the declared length ends is the data's, the length too short. Past the +window, the reader asks the file for the bytes after the declared length rather than reading the data, reading on as +far as white space and comments run, and they tell it whether `endobj` follows too. Any other stream — its length unconfirmed, or a `/Length` that gives none — is searched once for the first `endstream` after the start of its data, which ends it: in the bytes the parser holds, then in the file, up to the next object — the one the file's index places as the file wrote it (as the reader rebuilt it when the document opened, if the file wrote none), @@ -52,24 +56,26 @@ stream of a classic file's last object, which no object follows, is searched thr end of the file. With no `endstream` before the stop, the object's `endobj` plays its part. A declared length that ends before the stop, -and that the object's `endobj` follows, is kept: the stream lost only its `endstream`. Otherwise, the object's own -`endobj`, when only white space separates it from the stop, ends the data, the end-of-line before it left out. Both are -reported as `stream.truncated`. Without one, a declared length that ends before the stop is kept, and whether the -object ends with its `endobj` is not judged; and the data otherwise runs to the stop, a declared length past it cut -there, the object judged to lack its `endobj` unless what is taken holds one. Both are reported as +and that the object's `endobj` follows as an `endstream` would, is kept: the stream lost only its `endstream`. Otherwise, +the object's own `endobj`, when only white space and comments separate it from the stop, ends the data, the end-of-line +before it left out. Both are reported as `stream.truncated`. Without one, a declared length that ends before the stop is +kept, and whether the object ends with its `endobj` is not judged; and the data otherwise runs to the stop, a declared +length past it cut there, the object judged to lack its `endobj` unless what is taken holds one. Both are reported as `stream.length-invalid`, but data that runs to the end of the file as `stream.truncated`. A `/Length` that names an object one of the reader's limits cut is no length the file gave: what the cut left of it — null, or `1` of a `15` it parted — is the guard's. An integer read before the cut is only a place to look for `endstream` at, and the data is otherwise found by its `endstream`, as for a stream with no length; nothing is said of the length, while a missing `endstream` is still the file's, and reported. One that names an object a guard left out of -the index is not judged either. +the index is not judged either: its data is found the same way, and a missing `endstream` reported as the file's. A cross-reference stream whose `/Length` the chain cannot read yet, which a valid file may hold, is searched as well, up to the next object the sections read so far place, or the end of the file, and its length read once the chain is. Its rows may spell an object header and end in a carriage return: no header stops its search, and a carriage return before the line feed that precedes `endstream` is kept. Data with no `endstream` before that stop ends nowhere the section -says, and its rows are not read. +says, and its rows are not read — the file's fault, unless the `/Length` leads to an object one of the reader's limits +cut: the length that would have said where the data ends is then the guard's, and so is the section, the chain ending +there. The search is not one of the reader's limits: a valid file's `endstream` follows its length, so only a damaged stream is searched — and a cross-reference stream whose `/Length` the chain cannot read yet —, each once. Where it stops depends diff --git a/docs/website/docs/concepts/lazy-reading.md b/docs/website/docs/concepts/lazy-reading.md index 3f8b0c80..e1e29fbe 100644 --- a/docs/website/docs/concepts/lazy-reading.md +++ b/docs/website/docs/concepts/lazy-reading.md @@ -16,12 +16,14 @@ Opening a document builds an index, finds the catalog, and stops there. 3. Builds a map from object number to a location: either a byte offset, or a position inside an object stream. Nothing is loaded meanwhile: a value a section refers to — a cross-reference stream's `/W` or `/Length` written as a reference, a `/Prev` — is read only where a section already read places it, so that - the map the chain gives is the file's own. A `/Length` no section read before places is read once the chain - is, and checked against the data the chain took up to the first `endstream` after the data's start, searched - for in the file up to the next object the sections read so far place, or its end — unless the section's own row - for that `/Length` lies among the rows the data holds: those rows spelled `endstream`, the object is read where - the row places it, without being kept, and the data is taken again to the length it gives when an `endstream` - confirms it. + the map the chain gives is the file's own. A `/Length` no section read before places, or that leads through + references to such an object, is read once the chain is, and checked against the data the chain took up to the + first `endstream` after the data's start, searched for in the file up to the next object the sections read so far + place, or its end; one that leads to an object `MaxObjectLength` cuts waits for the chain the same way, and is the + guard's: the guard is reached once the chain is read, and the length is not checked. Either way, when the section's + own row for that `/Length` — the row the reading of its rows keeps — lies among the rows the data holds, those + rows spelled `endstream`: the object is read where the row places it, without being kept, and the data is taken + again to the length it gives, or to the integer the guard's cut left of it, when an `endstream` confirms it. 4. Resolves the trailer's `/Root`, which reads the document catalog, and decodes the object stream that holds it when one does. When `/Root` leads to no catalog, it loads the objects the map holds, one after the other, until one is a catalog (`trailer.root-recovered`), and rebuilds the map when none is. @@ -57,9 +59,12 @@ An object the map does not hold is one the file does not define, and a reference specification says — or one a limit left out of the map, which reads as null too, until the limit is raised: the sections past `MaxXRefSectionCount`, the rows past `MaxXRefSectionLength` or past what `MaxStreamLength` reads or `MaxDecodedStreamLength` decodes of a cross-reference stream, a cross-reference stream whose `/W`, `/Index`, `/Size`, -`/Filter` or `/DecodeParms` leads to an object `MaxObjectLength` cuts and the sections older than it, the numbers past -`MaxIndexEntries`. That is no damage, and the map is not rebuilt for it. The map is rebuilt by scanning the whole file, -once at most, and never while the chain is read: +`/Filter` or `/DecodeParms` leads to an object `MaxObjectLength` cuts and the sections older than it — so too one whose +`/Length` leads to such an object, when no `endstream` ends its data before the next object or the end of the file, or +the rows past an `endstream` its rows spell, which only that length could tell from the end of its data —, the sections +older than a `/Prev` or an `/XRefStm` that leads to such an object, the numbers past `MaxIndexEntries`. That is no +damage, and the map is not rebuilt for it. The map is rebuilt by scanning the whole file, once at most, and never while +the chain is read: - at opening, when `startxref` cannot be found or read, when the section it names cannot be read, when the chain indexes nothing, or when `/Root` leads to no catalog and no object of the map is one; @@ -114,15 +119,18 @@ opening it reads less than 200,000 bytes of the file, and reading that content's 500,000 more. Nor are the bytes kept once they have arrived: neither the stream nor the object cache holds them, and each call reads -them again. A document held in memory — opened from bytes or from a stream — lends them where they lie, without a copy; -one opened from a path reads them from the file into an array of their length. What a call hands back is the caller's -to keep or let go, so that memory follows the streams being read or decoded, not those read before, and streams whose -data share a stretch of the file — objects that overlap — do not each hold a copy of it. What one call reads is bounded -by `PdfReaderLimits.MaxStreamLength`, 256 MB by default: past it the data is cut, its filters decode what was read as -data the reader cut, and `limit.stream-length` is reported; `RawLength` stays the length the file holds. +them again. A document held in memory — opened from bytes or from a stream — lends them where they lie, without a copy: +what a call hands back is a view of that memory, which keeps it alive for as long as it is held, after the document is +disposed too, and changes as it does. One opened from a path reads them from the file into an array of their length. +Either is the caller's to keep or let go, so that memory follows the streams being read or decoded, not those read +before, and streams whose data share a stretch of the file — objects that overlap — do not each hold a copy of it. What +one call reads is bounded by `PdfReaderLimits.MaxStreamLength`, 256 MB by default: past it the data is cut, its filters +decode what was read as data the reader cut, and `limit.stream-length` is reported; `RawLength` stays the length the +file holds. Finding where a stream's data ends reads little of it. The reader parses an object through a window of the file, and -confirms the declared `/Length` by the `endstream` that must follow it, after white space and comments, however many: +confirms the declared `/Length` by the `endstream` that must follow it, after white space and comments, however many — +a comment only once white space comes before it, since a `%` right where the declared length ends is the data's —: past the window, it asks the file for the bytes after the declared length rather than reading the data, and reads on as far as white space and comments run. A length nothing confirms, or a `/Length` that gives none, sends the reader on one search for the first `endstream` after the data's start: in the bytes the parser holds, then in the file, up to the @@ -137,8 +145,8 @@ When it is read or decoded, a stream works under the [limits](reader-limits.md) however long after opening — so with `ThrowOnLimit` set, `GetRawBytes()` and `Decode()` can throw long after `Open` returned. What decoding meets — a stream that lost its tail, a limit its filters reached — is reported when it is met, in the diagnostics you pass to `Decode`, or in the document's own when you pass none; data cut at `MaxStreamLength` is -reported in the document's own, once for each stream. A damaged stream nobody decodes is never reported, since nothing -reads it. +reported in the document's own once for each stream, as the reader reads it, and in those you pass to `Decode` each time +it decodes the cut data. A damaged stream nobody decodes is never reported, since nothing reads it. Holding a stream as an offset and a length is what will let a copy between documents — merging, assembling, stamping — move its **encoded** bytes as they are, with no decompress/recompress cycle in between, since nothing diff --git a/docs/website/docs/concepts/reader-limits.md b/docs/website/docs/concepts/reader-limits.md index bda180e3..e4f7e145 100644 --- a/docs/website/docs/concepts/reader-limits.md +++ b/docs/website/docs/concepts/reader-limits.md @@ -52,9 +52,10 @@ document it came from, however long after opening. A value read through a reference to an object a guard cut, or left out of the index, is the guard's too. What the cut left of it may be the start of what the file holds — `1` of `15`, `/Flate` of `/FlateDecode` —, so it is never judged as a value the file gave: the data of a stream whose `/Length` is so named is found by its `endstream`, and its length -is not judged; decoding stops at a filter so named, or one whose parameters are, and leaves the rest of the data -encoded, as past the most filters the reader runs; and a cross-reference stream whose values lead to an object a guard -cut ends the chain as a section a guard cut does. +is not judged, though an `endstream` the data lacks is still the file's to lack; decoding stops at a filter so named, or +one whose parameters are, and leaves the rest of the data encoded, as past the most filters the reader runs; and a +`/Prev` or an `/XRefStm` so named, or a cross-reference stream whose values lead to an object a guard cut, ends the +chain as a section a guard cut does. What a guard leaves out of the cross-reference index — the oldest sections of a long chain, the rows past a long table's bound, the objects past the most the index holds — is no damage either. Those objects read as null, unknown diff --git a/docs/website/docs/concepts/validation.md b/docs/website/docs/concepts/validation.md index e67637cb..980369ff 100644 --- a/docs/website/docs/concepts/validation.md +++ b/docs/website/docs/concepts/validation.md @@ -155,7 +155,9 @@ The document's [reader limits](reader-limits.md) apply to validation too. An obj limit is not a fault of the file, so a rule meeting one judges nothing of what it lacks or holds, or says, as information, that it could not check it whole, rather than calling the file broken; opening the document with a raised limit lets it check the rest. The same holds of a value that leads to such an object: a cross-reference stream whose -`/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` leads to one is said not checked rather than malformed, and an +`/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` leads to one is said not checked rather than malformed — so is one +whose `/Length`, which the chain could not read, leads to one where only that length could say where its data ends —; +a `/Prev` or an `/XRefStm` that leads to one ends the chain there rather than naming a section not found; and an object stream whose `/Filter` or `/DecodeParms` leads to one, or to an object a limit left out of the index, is left encoded, the objects it holds said not checked rather than lost ([#263](https://github.com/AdCodicem/AdCodicem.Pdf/issues/263)). The page tree rules take a node so cut to give what its diff --git a/docs/website/docs/guides/find-what-the-reader-repaired.md b/docs/website/docs/guides/find-what-the-reader-repaired.md index b988002a..75b6c464 100644 --- a/docs/website/docs/guides/find-what-the-reader-repaired.md +++ b/docs/website/docs/guides/find-what-the-reader-repaired.md @@ -93,6 +93,10 @@ if (document.GetObject(new PdfObjectId(12)) is PdfStream stream) } ``` +The list also receives, each time you decode the stream, `limit.stream-length` when its data runs past what the reader +reads of one stream; the document's `Diagnostics` hold that report too, once, since the reader makes it as it reads the +data. + ## Decide whether to accept the file Diagnostics say what the reader did, not whether the file is sound. To decide whether to accept it, validate it too: diff --git a/docs/website/docs/guides/tune-reader-memory.md b/docs/website/docs/guides/tune-reader-memory.md index 80638c0c..e7a551c9 100644 --- a/docs/website/docs/guides/tune-reader-memory.md +++ b/docs/website/docs/guides/tune-reader-memory.md @@ -32,8 +32,11 @@ its dictionary and where its data lies, not the data: its size does not weigh on A stream keeps none of its data: each call to `GetRawBytes` or `Decode` reads it again. Opened from a path, each call reads the data from the file into an array of its length; opened from a stream or from bytes, which the document holds -in memory, `GetRawBytes` lends the bytes where they lie, without a copy. When you use the same data more than once, -keep what one call gave you rather than call again: +in memory, `GetRawBytes` lends the bytes where they lie, without a copy, and so does `Decode` with data no filter +decodes. What is lent is a view of the document's memory: for as long as you hold it, it keeps that memory alive, after +`Dispose` too, and it changes if that memory does. Copy it before you reuse a buffer the document read in place. + +When you use the same data more than once, keep what one call gave you rather than call again: ```csharp using AdCodicem.Pdf.Objects; diff --git a/docs/website/docs/reference/diagnostics.md b/docs/website/docs/reference/diagnostics.md index 5833984c..91173e4c 100644 --- a/docs/website/docs/reference/diagnostics.md +++ b/docs/website/docs/reference/diagnostics.md @@ -74,7 +74,7 @@ The list `PdfDocument.Diagnostics` holds, in the order the entries were recorded | `Decode(stream, diagnostics)` | Where the reports go | |---|---| -| `diagnostics` given | Into it, and nowhere else | +| `diagnostics` given | Into it, each time the stream is decoded; the document's own `Diagnostics` also hold, once, the `limit.stream-length` the reader reports as it reads data `MaxStreamLength` cuts | | none, a stream read from a document | The document's own `Diagnostics` | | none, a stream built in memory | Nowhere: nothing is reported | @@ -101,18 +101,18 @@ Codes are stable: they are part of the public contract, because callers filter o | `header.missing` | The file has no `%PDF-` header in its first 4,096 bytes; it was read all the same, its offsets counted from its first byte | | `xref.rebuilt` | The index was rebuilt by scanning the whole file | | `xref.offset-adjusted` | An object, or a cross-reference section the chain names, was not where the file said, and was found nearby | -| `xref.section-missing` | A cross-reference section `/Prev` or `/XRefStm` names holds nothing that reads as a section, nor does any place nearby, lies outside the file, or is named by a value that is not an offset; what only it indexed is found by rebuilding the index when it is asked for. Reported at the offset the section was named at, or, when that offset lies outside the file or the value is not an offset, at the section whose trailer named it | -| `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, `/Size` or data cannot give rows, or whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` is a reference no section read before it places where it can be read without loading it, or whose `/Length` is such a reference and no `endstream` follows its data before the next object the sections read so far place, or before the end of the file, or none was looked for once the document's searches for `endstream` read as much of the file as they may — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message. A value that names an object one of the reader's limits cut is the guard's, not the file's: the section is not reported, nor looked for nearby, the chain ends there, nothing is rebuilt, what only that section and the older ones index reads as null, and the guard is reported once the chain is read | +| `xref.section-missing` | A cross-reference section `/Prev` or `/XRefStm` names holds nothing that reads as a section, nor does any place nearby, lies outside the file, or is named by a value that is not an offset; what only it indexed is found by rebuilding the index when it is asked for. Reported at the offset the section was named at, or, when that offset lies outside the file or the value is not an offset, at the section whose trailer named it. A `/Prev` or `/XRefStm` written as a reference that leads to an object one of the reader's limits cut names what the guard hides: no section is reported missing, nothing is rebuilt, the chain ends there, and the guard is reported once the chain is read | +| `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, `/Size` or data cannot give rows, or whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` is a reference no section read before it places where it can be read without loading it, or whose `/Length` is such a reference, or leads to one through others, and no `endstream` follows its data before the next object the sections read so far place, or before the end of the file, or none was looked for once the document's searches for `endstream` read as much of the file as they may — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message. A value that names an object one of the reader's limits cut, directly or through a chain of references — a `/Length` whose data no `endstream` ends before that stop among them —, is the guard's, not the file's: the section is not reported, nor looked for nearby, the chain ends there, nothing is rebuilt, what only that section and the older ones index reads as null, and the guard is reported once the chain is read | | `xref.chain-cycle` | The chain of previous sections looped: a `/Prev` or `/XRefStm` names an offset the chain has already named, whether it read a section there or not. Reported at that offset, or, when it lies outside the file, at the section whose trailer named it | | `xref.entry-out-of-range` | An object's entry places it outside the file — past its end, before its start, or past what a `long` holds once the header's offset is added —; reported with no position, the offset in the message, exactly. A section named outside the file is `xref.section-missing`'s, or the rebuild's when `startxref` names it | -| `stream.length-invalid` | A stream's `/Length` is not where its data ends: its `endstream` lies elsewhere and ends the data; or the `/Length` is no length — absent, not a non-negative integer, or naming an object the file lacks or that could not be read — and the `endstream` ends the data; or no `endstream` lies before the next object or the end of the file, no `endobj` of its object ends the data, and the data is the declared length, when it ends before that stop, or runs up to the next object; or the document's searches for `endstream` have read as much of the file as they may, and the data is taken without a search. A `/Length` that names an object a guard cut, or left out of the index, is not judged. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case. A cross-reference stream whose `/Length` the chain could not read, its data taken up to its first `endstream`, has it read once the chain is: a `/Length` that is no length, or that no `endstream` confirms, is reported as any stream's, and one an `endstream` past the one the chain stopped at confirms is reported with the rows the chain left unread. Where the section's own row for that `/Length` lies among the rows the data holds, and gives a length an `endstream` confirms, the data is taken again to it before the rows are read, and nothing is reported; nor is anything of a `/Length` a guard cut or left out of the index | +| `stream.length-invalid` | A stream's `/Length` is not where its data ends: its `endstream` lies elsewhere and ends the data; or the `/Length` is no length — absent, not a non-negative integer, or naming an object the file lacks or that could not be read — and the `endstream` ends the data; or no `endstream` lies before the next object or the end of the file, no `endobj` of its object ends the data, and the data is the declared length, when it ends before that stop, or runs up to the next object; or the document's searches for `endstream` have read as much of the file as they may, and the data is taken without a search. A `/Length` that names an object a guard cut, or left out of the index, is not judged, though a missing `endstream` is reported all the same. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case. A cross-reference stream whose `/Length` the chain could not read, its data taken up to its first `endstream`, has it read once the chain is, unless a guard cut that data: a `/Length` that is no length, or that no `endstream` confirms, is reported as any stream's, and one an `endstream` past the one the chain stopped at confirms is reported with the rows the chain left unread. Where the section's own row for that `/Length` lies among the rows the data holds, and gives a length an `endstream` confirms, the data is taken again to it before the rows are read, and nothing is reported; nor is anything of a `/Length` a guard cut or left out of the index | | `stream.truncated` | A stream has no `endstream` before the next object or the end of the file, and its object's `endobj` ends its data — following the declared length, which is kept, or ending the object before that stop —; or neither, no declared length ends before the end of the file, and its data runs to it. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case | | `stream.self-reference` | An object stream's dictionary names an object the stream holds — as its `/Type`, `/Length`, `/N`, `/First`, a filter or a parameter —: that object reads as null while the stream is decoded, and the stream is decoded without it | | `object-stream.member-moved` | An object is in the object stream its entry names, at another index than the entry gives, and was read where the stream's header lists it. Reported where the stream's data starts, the object, the stream and both indexes in the message | | `object-stream.unreadable` | What an object stream — a stream an entry of the index names as one — says of itself cannot be believed: it is not of `/Type /ObjStm`, its `/N` is absent or no non-negative integer, or, when it declares objects, its `/First` is, or lies past its decoded data, or its header ends or breaks before listing as many objects as `/N` declares — a pair that is not two integers, that gives a number no object can have or an offset no member can start at, whose offset is not past the one before it, or whose member would start at or past the end of the decoded data, unless a guard cut it. The header is read as far as it goes, whatever its `/N` claims: the objects listed before the fault are read, the others cannot be read from it. Reported where the stream's data starts, the stream and its fault in the message. An object stream whose `/Filter`, `/DecodeParms` or a value of a step's parameters names an object a guard cut, or left out of the index, is left encoded and not reported: its objects are the guard's, unknown until it is lifted | | `object-stream.member-stream` | An object an object stream holds is written as a stream, or holds one, which ISO 32000-1 (7.5.7) forbids: it reads as null, and nothing of it is decoded. Reported once for each object and each definition of its object stream, where the stream's data starts, the object, the stream and the byte of its decoded data where the object starts in the message | | `syntax.unexpected-token` | A token stood where the syntax does not allow it: where a value or a dictionary key was expected, after a key that has no value, or closing an array with a dictionary's end; it was read as null, skipped, or it ended what it stood in | -| `syntax.truncated-object` | An object ended before it was whole: the file, or an object stream's decoded data, ended in the middle of it — a value missing, or an array, a dictionary or a string it opened never closed —, or an `endobj`, or the next object's header, stood where an element, a key or a value should be, inside an array or a dictionary it opened. What was read is kept: a string takes the bytes to the end of the data, and an `endobj` ends every container it finds open, as does the next object's header, which an object that lost its `endobj` too meets first, and which leaves the object without its `endobj`. A header is `N G obj` as the search for a stream's `endstream` stops at one: digits alone, N from 1 to 2,147,483,647 and G from 0 to 65,535, white space alone between them — no comment —, and `obj` a whole keyword; tokens that read as one inside a container that closes after them end it there all the same, and the object's `endobj` is judged missing. An object stream's member, which no header follows, and a value standing alone, which is no container, never end at one. Reported once, for the innermost construct left open, where it opens — where the value would start, when none is open; past the depth the reader follows, the container it skipped, which holds what it did not follow —, the constructs around it counted in the message; inside an object an object stream holds, where the stream's data starts, the member and the byte in the message. A member is read to the next member's start, the last to the end of the decoded data: running into either, it is cut short there, and the message says which. The edge of a window the reader grows is no end of the data, and a cut a guard made is the guard's — a member read past `MaxObjectLength` is `limit.object`'s | +| `syntax.truncated-object` | An object ended before it was whole: the file, or an object stream's decoded data, ended in the middle of it — a value missing, or an array, a dictionary or a string it opened never closed —, or an `endobj`, or the next object's header, stood where an element, a key or a value should be, inside an array or a dictionary it opened. What was read is kept: a string takes the bytes to the end of the data, and an `endobj` ends every container it finds open, as does the next object's header, which an object that lost its `endobj` too meets first, and which leaves the object without its `endobj`. A header is `N G obj` as the search for a stream's `endstream` stops at one: digits alone, N from 1 to 2,147,483,647 and G from 0 to 65,535, white space alone between them — no comment —, and `obj` a whole keyword; tokens that read as one inside a container that closes after them end it there all the same, and the object's `endobj` is judged missing. An object stream's member, which no header follows, and a value standing alone, which is no container, never end at one. A trailer, which is no object, ends at one all the same, and the message says the header came inside what it left open: `An object header came inside a dictionary, which was never closed.` Reported once, for the innermost construct left open, where it opens — where the value would start, when none is open; past the depth the reader follows, the container it skipped, which holds what it did not follow —, the constructs around it counted in the message; inside an object an object stream holds, where the stream's data starts, the member and the byte in the message. A member is read to the next member's start, the last to the end of the decoded data: running into either, it is cut short there, and the message says which. The edge of a window the reader grows is no end of the data, and a cut a guard made is the guard's — a member read past `MaxObjectLength` is `limit.object`'s | | `syntax.depth-exceeded` | Nesting went deeper than the reader will follow | | `syntax.number-out-of-range` | A number is beyond what a real can hold — its magnitude rounds past the largest a double holds, about 1.8 × 10³⁰⁸ —: it was read as null, so a dictionary holds no entry for its key, and an array holds a null in its place. Reported where the number starts — where its object stream's data starts, for a number inside an object one holds —, quoted in the message. An integer past the range of a `long` is no fault of its own: it reads as the real nearest to it | | `syntax.hex-string-invalid` | A hexadecimal string holds bytes that are neither hexadecimal digits nor white space, the only bytes ISO 32000-1 (7.3.4.3) lets it hold — a `%` among them, which is no comment there —: they were skipped, as pdf.js and pdfium skip them, where PDFBox stops at the first, MuPDF and poppler read each as the digit 0, and qpdf and pypdf lose the string. Reported once for each string, where its first such byte lies, how many there are in the message; inside an object an object stream holds, where the stream's data starts, the member and the byte in the message | @@ -124,7 +124,7 @@ Codes are stable: they are part of the public contract, because callers filter o | `filter.checksum-mismatch` | A Flate stream decoded to its end, but its zlib checksum disagrees with what it decoded to: all of it was kept, and some of it may be wrong | | `filter.unsupported` | The file names a filter the library does not implement, or an element of `/Filter` names no filter at all, or `/Filter` is neither a name nor an array; decoding stopped there, with what the filters before it decoded. A `/Filter` that is null is one the stream does not have, and is not reported. Nor is a `/Filter`, an element of it, a `/DecodeParms` or a value of a step's parameters that names an object a guard cut, or left out of the index — directly or through a chain of references —: what the cut left of it, such as a name it parted, is the guard's, and decoding ends there, as it does at the element past `MaxFilterChainLength`, with what the filters before it decoded; nothing more is reported, the guard having been reported where it was reached | | `limit.decoded-stream` | A filter decodes to more than `MaxDecodedStreamLength`, or the filters before the last pass along more than it together; decoding stopped there, and the stream comes back as the filters after the cut decode what was kept within the bound | -| `limit.stream-length` | A stream's encoded data, as the file holds it, is longer than `MaxStreamLength`, which bounds what is read of it at once: the data within the bound was read, decoded as data the reader cut — what the filters lose past the cut is no damage — or handed back as it is where decoding hands data back undecoded, as with no filter, an image filter or a filter the library does not know. `Data.Length` and `RawLength` stay the length the file holds. Reported once for each stream, where its data starts, when its data is first read — by `GetRawBytes` or by decoding —, in the document's diagnostics whatever diagnostics `Decode` is given | +| `limit.stream-length` | A stream's encoded data, as the file holds it, is longer than `MaxStreamLength`, which bounds what is read of it at once: the data within the bound was read, decoded as data the reader cut — what the filters lose past the cut is no damage — or handed back as it is where decoding hands data back undecoded, as with no filter, an image filter or a filter the library does not know. `Data.Length` and `RawLength` stay the length the file holds. Reported where the stream's data starts: once in the document's diagnostics, when its data is first read — by `GetRawBytes` or by decoding —, and in diagnostics of your own passed to `Decode` with each decoding, as `limit.decoded-stream` is | | `limit.object` | An object is longer than `MaxObjectLength`, its stream data aside; the part within it was parsed. An object an object stream holds is bounded alike, and reported once for each definition of its object stream, where that stream's data starts, the object, the stream and the byte of its decoded data where the object starts in the message | | `limit.xref-section-length` | A classic cross-reference table is longer than `MaxXRefSectionLength`; the entries within it were read, and its trailer past them; the objects the rows past them place read as null, and the chain is not followed further | | `limit.xref-section-count` | The chain of cross-reference sections is longer than `MaxXRefSectionCount`; the newest were read, and what only the older ones index reads as null. Reported where the chain goes on, or, when that lies outside the file, at the section whose `/Prev` names it, the offset in the message | @@ -155,30 +155,36 @@ Warning syntax.key-repeated at 412: The dictionary gives the key /Filter more th ### Streams whose length is wrong -A declared length is confirmed by an `endstream` after it, white space and comments between, however many. Any other -stream — its length unconfirmed, or none to confirm — is searched once for the first `endstream` after the start of its -data, up to the next object: the first object header the file's bytes hold after that start, or the next object the -file's index places, whichever is nearer. Each case is reported where the data starts, and each stream once, however -often it is parsed again — after the cache let it go, or the index was rebuilt. How the reader finds where such a -stream ends, and why, is explained in [Diagnostics](../concepts/diagnostics.md#where-a-stream-ends). +A declared length is confirmed by an `endstream` after it, white space and comments between, however many — a comment +only once white space comes before it: a `%` right where the declared length ends is the data's, and leaves the length +unconfirmed. Any other stream — its length unconfirmed, or none to confirm — is searched once for the first `endstream` +after the start of its data, up to the next object: the first object header the file's bytes hold after that start, or +the next object the file's index places, whichever is nearer. Each case is reported where the data starts, and each +stream once, however often it is parsed again — after the cache let it go, or the index was rebuilt. How the reader +finds where such a stream ends, and why, is explained in [Diagnostics](../concepts/diagnostics.md#where-a-stream-ends). | What follows the data | Code | The data taken | |---|---|---| | An `endstream` after the declared length | None | The declared length | | No such `endstream`, or no length to confirm; an `endstream` before the next object or the end of the file | `stream.length-invalid` | Up to that `endstream`, the end-of-line before it left out | -| No `endstream` before that stop; the object's `endobj` after the declared length, which ends before the stop, white space and comments between | `stream.truncated` | The declared length | -| No `endstream` before the stop; the object's own `endobj` ends the object there, only white space between them | `stream.truncated` | Up to that `endobj`, the end-of-line before it left out | +| No `endstream` before that stop; the object's `endobj` after the declared length, which ends before the stop, white space and comments between, as before an `endstream` | `stream.truncated` | The declared length | +| No `endstream` before the stop; the object's own `endobj` ends the object there, only white space and comments between them | `stream.truncated` | Up to that `endobj`, the end-of-line before it left out | | None of the above; a declared length that ends before the stop | `stream.length-invalid` | The declared length; whether the object ends with its `endobj` is not judged | | None of the above; no declared length that ends before the stop | `stream.length-invalid` at the next object, `stream.truncated` at the end of the file | Up to the stop; the object lacks its `endobj`, unless what is taken holds one | | Not searched: the document's searches for `endstream` have read as much of the file as they may, and the bytes read with the object hold neither an `endstream` nor an object header | `stream.length-invalid`, the message saying so | The declared length, when it ends before the stop; otherwise up to the stop, the file past those bytes not searched for a header | +A file that ends on the `stream` keyword, or on the carriage return after it, holds a stream whose data starts where the +file ends and holds nothing: its missing `endstream` is reported as after a line feed, as the table gives it. + A `/Length` that names an object one of the reader's limits cut — directly or through a chain of references — gives no length the file wrote: what the cut left of it, null or an integer it parted, is the guard's, which reported it. An integer read before the cut is only a place to look for `endstream` at — the data ends there when one follows it —, never kept as a length; the data is otherwise found by its `endstream`, as for a stream with no length, and nothing is said of the length. A missing `endstream` is still the file's, and is reported, the message naming the object the guard -cut. A `/Length` that names an object a guard left out of the index is not judged either, and nothing is reported of -where its stream ends. +cut. A `/Length` that names an object a guard left out of the index, directly or through a chain of references, is not +judged either: the data is found by its `endstream` the same way, and a missing one is the file's, the message naming the +object the guard left out — `The stream's /Length names object 21 0, which one of the reader's limits left out of the +index, and no endstream follows its data`, then where the search stopped and what was taken. ```text Warning stream.length-invalid at 5501: The stream declared 19954 bytes but ended after 19952. diff --git a/docs/website/docs/reference/object-model.md b/docs/website/docs/reference/object-model.md index 931d9f05..db3625a1 100644 --- a/docs/website/docs/reference/object-model.md +++ b/docs/website/docs/reference/object-model.md @@ -96,12 +96,13 @@ read or decoded, and for as long as the caller keeps what that gave. | `Dictionary` | The stream's dictionary | Nothing: it was read with the stream | | `RawLength`, `Data.Length` | The count of the stream's encoded bytes in the file: its `/Length` where an `endstream` confirms it, otherwise as far as the reader takes the data to run ([Where a stream ends](../concepts/diagnostics.md#where-a-stream-ends)). Data longer than `MaxStreamLength` is counted whole | Nothing: found when the stream is parsed | | `Data.Position` | The offset in the file the data starts at | Nothing | -| `GetRawBytes()`, `Data.GetBytes()` | The encoded bytes, as the file holds them, up to `MaxStreamLength`: past it, the bytes within the bound, and `limit.stream-length` reported once for the stream | A read, each call: from a document opened from bytes or from a stream, the bytes lent where they lie, without a copy; from one opened from a path, read from the file into an array of their length | -| `Decode()` | The data the stream's filters decode, as far as they go: decoding stops at an image filter, at a filter the library does not know, and where the [reader limits](reader-limits.md) end it | A read, as `GetRawBytes()` makes, then what each filter produces; data no filter decodes comes back as it was read | +| `GetRawBytes()`, `Data.GetBytes()` | The encoded bytes, as the file holds them, up to `MaxStreamLength`: past it, the bytes within the bound, and `limit.stream-length` reported once for the stream, in the document's diagnostics | A read, each call: from a document opened from bytes or from a stream, the bytes lent where they lie, without a copy — a view of that memory, which keeps it alive for as long as you hold it, and changes as it does; from one opened from a path, read from the file into an array of their length | +| `Decode()` | The data the stream's filters decode, as far as they go: decoding stops at an image filter, at a filter the library does not know, and where the [reader limits](reader-limits.md) end it — a `/Filter`, a `/DecodeParms` or a parameter that leads to an object a limit cut or left out of the index among them. Data cut at `MaxStreamLength` is decoded as far as it was read, and `limit.stream-length` reported in the diagnostics you pass, each time | A read, as `GetRawBytes()` makes, then what each filter produces; data no filter decodes comes back as it was read | +| `HasImageFilter()` | Whether `Decode()` hands back an encoded image: whether `/Filter` names an image filter before anything that ends decoding whatever the data holds — a filter the library does not know, an element that names no filter, a filter past `MaxFilterChainLength`, a value or parameter of the filters' that leads to an object a limit cut or left out of the index | None of the data: what decoding reads before the image filter — the filters' names, and the parameters of a Flate or LZW step before it —, which can load an object, and decode the object stream that holds it, as decoding would | Opened with `ThrowOnLimit`, a document's streams throw a `PdfLimitExceededException` from `GetRawBytes()` and `Decode()` instead, each time their data runs past `MaxStreamLength`; `Decode()` throws at the other limits it reaches -too. No call's result is kept for the next: a caller that uses the data twice keeps what one call gave, rather than call +too, and `HasImageFilter()` when loading a value the filters read reaches one. No call's result is kept for the next: a caller that uses the data twice keeps what one call gave, rather than call again ([Tune the reader's memory](../guides/tune-reader-memory.md#read-a-streams-data-once)). Data built with `PdfStreamData.FromMemory` is held by the stream, whatever its length, and handed back as it was given; @@ -112,4 +113,6 @@ its `Position` is -1. An object keeps what it holds once its document is disposed: a dictionary's direct values, an array's items, a stream's `RawLength`, every object's `ToString()`. What reaches back into the document throws `ObjectDisposedException`, however the document was opened and whether or not it was read before: `Resolve()` on a reference, and every accessor that -resolves one, such as `GetDictionary`; `GetRawBytes()` and `Decode()` on a stream read from the file. +resolves one, such as `GetDictionary`; `GetRawBytes()` and `Decode()` on a stream read from the file, and +`HasImageFilter()` when it has a value to load. Raw bytes taken before from a document held in memory stay a view of +that memory: they keep it alive for as long as they are held. diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index fa9c48e9..6326dbce 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -67,10 +67,11 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. that entry was not checked. - A stream whose data runs past `MaxStreamLength` is read up to the bound each time its data is asked for, by `GetRawBytes` or by decoding, and its filters decode what was read as data the reader cut: that it runs out is not - reported as damage. Data that decoding hands back as it is comes back as far as the bound. The guard is reported once for - each stream, in the document's own diagnostics whatever you pass to `Decode`, at the offset its data starts at; with - `ThrowOnLimit` each read throws. In an object stream so cut, an object that lies past the cut is the guard's, unknown - until you raise the limit, as what any limit leaves out is: + reported as damage. Data that decoding hands back as it is comes back as far as the bound. The guard is reported at the + offset the stream's data starts at: once in the document's own diagnostics, as the reader reads the data, and, as the + bound on decoded data is, in the diagnostics you pass to `Decode` each time it decodes the stream. `GetRawBytes` takes + no diagnostics: the document's hold its report. With `ThrowOnLimit` each read throws. In an object stream so cut, an + object that lies past the cut is the guard's, unknown until you raise the limit, as what any limit leaves out is: ```text Warning limit.stream-length at 1904: A stream's data runs past 256 MB; only what lies within it was read. Raise @@ -110,20 +111,26 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. guard's, and nothing is reported of it but the guard. The data of a stream whose `/Length` is so named is found by the search for its `endstream`, and its length is not judged: an integer the cut left, which may be the first digits of a longer one, is only a place to look for `endstream` at. No `endstream` after the data of a stream whose - `/Length` names an object a guard cut is still the file's fault, and reported. Decoding stops at a filter so named, - or at the filter whose parameters are: the filters before it run, and the stream comes back as they decoded it, - still encoded by the rest, as past `MaxFilterChainLength`. An object stream the reader meets so is read as one a - guard cut before its data: its objects are unknown until you raise the limit, and when the catalog is among them the - reader does not look for another. A cross-reference stream whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` - leads to an object `MaxObjectLength` cuts while the chain is read is a section the guard cut: the chain ends there, - the index is not rebuilt, and the guard is reported once the chain is read, or thrown by `Open`. + `/Length` names an object a guard cut, or left out of the index, is still the file's fault, and reported; the report + names the object at the end of the chain of references. Decoding stops at a filter so named, or at the filter whose + parameters are: the filters before it run, and the stream comes back as they decoded it, still encoded by the rest, + as past `MaxFilterChainLength`. `HasImageFilter` reads what decoding reads before the image filter, the parameters of + a Flate or LZW filter that comes before it included, and says false when such a value ends the chain there; under + `ThrowOnLimit`, loading one throws where decoding would. An object stream the reader meets so + is read as one a guard cut before its data: its objects are unknown until you raise the limit, and when the catalog + is among them the reader does not look for another. While the chain is read, a `/Prev` or an `/XRefStm` that leads to + an object `MaxObjectLength` cuts, a cross-reference stream whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` + does, and one whose `/Length` does when no `endstream` ends its data, are the guard's, as a section it cut is: the + chain ends there, the older sections are not read, the index is not rebuilt, and the guard is reported once the chain + is read, or thrown by `Open`. - A stream read from a document decodes under **that document's** limits, however long after opening you decode it. A stream you build in memory decodes under `PdfReaderLimits.Default`. - A classic trailer that ends within the window its cross-reference table was read through is read whole, whatever its length. `MaxTrailerLength` bounds how far the reader follows one past that window, and a cross-reference stream's dictionary, not its data: the data of one whose `/Length` the reader cannot read while it reads the chain is searched for its `endstream` in the file, up to the next object the sections read so far place, or the end of the file. Data - no `endstream` ends before there makes its section malformed, a fault of the file and not of a guard. + no `endstream` ends before there makes its section malformed, a fault of the file and not of a guard — unless its + `/Length` leads to an object `MaxObjectLength` cuts, which makes the section the guard's, as above. - Until the reader decodes streams a piece at a time, a stream that decodes past about 2 GB is cut there, and reported, even under `Unbounded`. A stream's data is read in one piece, and data past about 2 GB is read up to there, and reported under `limit.stream-length`, even under `Unbounded`. @@ -151,6 +158,7 @@ The exception comes from whichever operation reaches the limit: | `GetObject`, or resolving a reference | When an object is too long, or the object stream that holds it runs past `MaxStreamLength`, decodes too far or names too many filters; when the read rebuilds the index, at any limit the rebuild reaches, once the rebuild is finished — the first reached: a limit the rebuild's scan reaches before the read's own, the read's own before one finishing the rebuild reaches; and each time it looks up an object a full index left out | | `GetRawBytes` | When the stream's data runs past `MaxStreamLength`, each time it is read | | `Decode` | When a stream's data runs past `MaxStreamLength`, the stream decodes too far, or its decoding reaches the filter past `MaxFilterChainLength` | +| `HasImageFilter` | When loading a value decoding reads before the image filter — the `/Filter` entry, a filter's parameters — reaches a limit, as resolving a reference does | | `PdfValidator.Validate` | As it reads, like any other caller; and before its first rule, at any limit the rebuild it may run then reaches, once that rebuild is finished | A document whose index had to be rebuilt finishes the rebuild before throwing, so it stays usable afterwards. diff --git a/docs/website/docs/reference/validation-rules.md b/docs/website/docs/reference/validation-rules.md index 0b420604..c7f888f0 100644 --- a/docs/website/docs/reference/validation-rules.md +++ b/docs/website/docs/reference/validation-rules.md @@ -45,14 +45,14 @@ References are to ISO 32000-1:2008. | `file.trailer-malformed` | Error | A trailer is not a well-formed dictionary: the `trailer` keyword is not followed by one, or the reader read it despite syntax errors — a key that is not a name, a dictionary or a string left unclosed, a hexadecimal string holding bytes that are neither digits nor white space, a name whose number sign is no escape, a key given twice —, so an entry may have been lost with the syntax, or read otherwise than its writer meant. A cross-reference stream's dictionary is held to the same. | Each trailer of the chain, as the reader parsed it | | `file.root-invalid` | Error | The trailer's `/Root` is missing or does not lead to a document catalog (Table 15), a value that is not a reference included — unless it is a catalog written in the trailer itself, which the reader takes as it is ([#111]). The reader then looks for an object of `/Type /Catalog` among those the index holds — and rebuilds the index only when none is one —: the catalog it finds is its choice, not the file's. Not judged when the trailer could not be read at all, nor when the index was rebuilt as the document opened, nor when one of the reader's limits left the catalog out of the index as the document opened, which `xref.checked-in-part` says, or cut the object `/Root` names — through a chain of references too, and in an object stream whose cut leaves it unread, past the cut data or in a stream the guard made unreadable ([#259]), or in one whose filters end at a `/Filter`, a `/DecodeParms` or a parameter that leads to an object a guard cut or left out of the index, which leaves its data encoded ([#263]) — or the trailer a catalog written there was read from, which the reader then takes for the catalog as it read it, looking for no other, and its `limit.*` report names ([#175]) — whatever a rebuild found since —, nor a `/Root` the trailers lack where one of the reader's limits kept a trailer from being read whole — cut, or older than where the chain stopped, at the section count or at a section whose rows a limit cut or filled the index with —: it may lie there, and the reader rebuilds nothing for it. Rows a limit cut that run on to no trailer lack one: the file's, which a rebuild answers. The catalog the finding names is the one the reader took last, in the rebuild run before the first rule if there was one. | The trailer the chain gave, merged as readers merge it, and `/Root` before the reader looked for a catalog | | `file.size-wrong` | Warning | A section's `/Size` is missing, or is not one more than the highest object number it and the sections it updates use (Tables 15 and 17, 7.5.6, 7.5.8.4). A hybrid file's cross-reference stream meets one of Table 17's two readings or the other — one more than the highest number it and the sections it updates use, or the `/Size` of the table that names it —, and either is accepted. A `/Size` that leaves in-use objects numbered above it is `xref.object-past-size`'s. A `/Size` written as a reference, or as a real, is said to be. Not judged when the chain lost a section, or one of the reader's limits cut it. | Each section's trailer, and the highest number each section indexes | -| `file.trailer-value-wrong` | Warning | A value a section's trailer gives the reader to read the chain is written as the specification does not allow, and read all the same, as other readers read it: a reference where it must be direct, read where a section read before it places the object; a real with no fractional part where it must be an integer, read as the integer it equals ([#182], [#215]). A cross-reference stream's `/Type`, `/Index`, `/Prev` and `/W`, the elements of `/Index` and `/W`, and its `/Filter` and `/DecodeParms` are direct (7.5.8.2); in a file that declares PDF 2.0, so are its `/Length` and the elements and entries of its `/Filter` and `/DecodeParms` (an erratum of ISO 32000-2, pdf-issues #246). A classic trailer's `/Prev` is direct (Table 15); an `/XRefStm` may be indirect (Table 19). `/W`, `/Index`, `/Prev`, `/XRefStm`, a stream's `/Length` and its `/Predictor`, `/Colors`, `/BitsPerComponent`, `/Columns` and `/EarlyChange` are integers. One finding per value, which says both faults of a reference that leads to a real. A value the rows are read with that the reader could not read so is the section's fault — `xref.section-malformed`, `xref.section-not-found` —, unless it leads to an object one of the reader's limits cut, which leaves the section to `xref.checked-in-part` ([#263]), and `/Size` is `file.size-wrong`'s; a `/Type`, or a `/Length` read once the chain is, is reported here whether or not it could be read. | Each section's trailer, as the file wrote it, and the objects its references name, through the index the reader settled on before the first rule | +| `file.trailer-value-wrong` | Warning | A value a section's trailer gives the reader to read the chain is written as the specification does not allow, and read all the same, as other readers read it: a reference where it must be direct, read where a section read before it places the object; a real with no fractional part where it must be an integer, read as the integer it equals ([#182], [#215]). A cross-reference stream's `/Type`, `/Index`, `/Prev` and `/W`, the elements of `/Index` and `/W`, and its `/Filter` and `/DecodeParms` are direct (7.5.8.2); in a file that declares PDF 2.0, so are its `/Length` and the elements and entries of its `/Filter` and `/DecodeParms` (an erratum of ISO 32000-2, pdf-issues #246). A classic trailer's `/Prev` is direct (Table 15); an `/XRefStm` may be indirect (Table 19). `/W`, `/Index`, `/Prev`, `/XRefStm`, a stream's `/Length` and its `/Predictor`, `/Colors`, `/BitsPerComponent`, `/Columns` and `/EarlyChange` are integers. One finding per value, which says both faults of a reference that leads to a real. A value the chain is read with that the reader could not read so is the section's fault — `xref.section-malformed`, `xref.section-not-found` —, unless it leads to an object one of the reader's limits cut, where the chain ends for the limit, a section whose rows need it left to `xref.checked-in-part` ([#263]), and `/Size` is `file.size-wrong`'s; a `/Type`, or a `/Length` read once the chain is, is reported here whether or not it could be read. | Each section's trailer, as the file wrote it, and the objects its references name, through the index the reader settled on before the first rule | ### Cross-references | Rule | Severity | Meaning | Reads | |---|---|---|---| -| `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]); or a cross-reference stream whose `/Length` is such a reference, and whose data no `endstream` ends before the next object the sections read so far place, or the end of the file — or was not searched for one, the document's searches for `endstream` having read as much of the file as they may ([#220]). Rows that spell `endstream` end such a stream's data there, and the section lacks the rows past them, unless the section's own row for that `/Length`, among the rows before them, places an object that gives a longer length an `endstream` confirms: the data is then taken to it. A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. Nor is a cross-reference stream whose value the rows are read with leads to an object a limit cut: the section counts as cut there, the chain ends at it, nothing is rebuilt, and `xref.checked-in-part` says so ([#263]). | Each section of the chain | -| `xref.section-not-found` | Error | A section a `/Prev` or an `/XRefStm` names is neither there nor within 512 bytes of it, or what names it is not an offset — tiff2pdf writes `/Prev 576066 0 R`. The objects only it indexes are missing from the index. The section `startxref` names is `file.startxref-wrong`'s. | Each section of the chain | +| `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]); or a cross-reference stream whose `/Length` is such a reference, or leads through references to an object no section read before it places, and whose data no `endstream` ends before the next object the sections read so far place, or the end of the file — or was not searched for one, the document's searches for `endstream` having read as much of the file as they may ([#220]). Rows that spell `endstream` end such a stream's data there, and the section lacks the rows past them, unless the section's own row for that `/Length` — the row the reading of its rows keeps, one of a reserved type ignored —, among the rows before them, places an object that gives a longer length an `endstream` confirms: the data is then taken to it. A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. Nor is a cross-reference stream whose value the rows are read with leads to an object a limit cut: the section counts as cut there, the chain ends at it, nothing is rebuilt, and `xref.checked-in-part` says so ([#263]). Nor is one whose `/Length`, which the chain could not read, leads to an object `MaxObjectLength` cuts, where only that length could say where the data ends: with no `endstream` before the next object or the end of the file, the section counts as cut, as above; when its rows spell one, the integer the cut left of the object the section's own row places is where an `endstream` is looked for, the data taken to it when one confirms it, and the rows past the spelled one the limit's otherwise ([#224]). | Each section of the chain | +| `xref.section-not-found` | Error | A section a `/Prev` or an `/XRefStm` names is neither there nor within 512 bytes of it, or what names it is not an offset — tiff2pdf writes `/Prev 576066 0 R`. The objects only it indexes are missing from the index. The section `startxref` names is `file.startxref-wrong`'s. A `/Prev` or an `/XRefStm` written as a reference that leads to an object `MaxObjectLength` cuts names no section missing: the chain ends there for the limit, the sections older than it unread ([#263]). | Each section of the chain | | `xref.section-shifted` | Warning | A section a `/Prev` or an `/XRefStm` names starts a few bytes from where it is named, and the reader found it there. | Each section of the chain | | `xref.chain-loop` | Error | The chain names a section it has already read, or an offset it has already named where it read none, in the file or outside it: the reader stops there, and the section the chain should have gone on to is unknown, what only it indexed found, if at all, by rebuilding the index — as for `xref.section-not-found`. | The chain | | `xref.entry-broken` | Error | An in-use entry places its object neither at its offset nor within 512 bytes of it, in an object stream whose header does not list it, or in one where it is written as a stream or holds one, which an object stream cannot hold (7.5.7), and which reads as null — the index the reader rebuilt included, where it places such an object and the file's own index places nothing, or the part of it the reader read where one of its limits stopped the chain short ([#161]) — as a load decides it: the header read at the offset after white space and comments, however long, a whole token no header begins with ruling one out at once, and the one found near it confirmed by what a load parses there ([#175], [#256]). An offset outside the file — past its end, before its start, or past what a `long` holds once the header's offset is added — is looked for nearby as a load looks: an object found there is `xref.entry-shifted`'s. One finding per entry, located at the object and the offset the entry gives, or at the object alone when that offset lies outside the file. | Each object's header, as a load reads it, a few dozen bytes in a sound file; each object stream's header, and each member whose bytes hold `stream`, parsed as a load parses it | @@ -87,7 +87,7 @@ it is checked: the exception above judges the index the reader rebuilt, and the | Rule | Severity | Meaning | Reads | |---|---|---|---| | `object.reference-missing` | Warning | An object reachable from the trailer refers to an object the file does not define — none in its index, a free entry, or object 0, which heads the free list and is never in use (7.5.4) —, which is null (7.3.10): the reader reads it so, rebuilding nothing, and what the reference was to give is lost. One finding per object that holds such references, naming the first. An object one of the reader's limits left out of the index is unknown rather than missing, and is `xref.checked-in-part`'s. An object the index holds and the reader cannot produce is `xref.entry-broken`'s or `xref.object-stream-broken`'s — but one a rebuild serves from another object stream than the chain's index places it in, or from another definition of that stream, which does not list it or holds it written as a stream, and so loses, which no finding names yet ([#276]) —, a kid of the page tree `page-tree.kid-invalid`'s — one in a node's `/Kids`, an array of kids of its own or a root written in the catalog the walk read whole, not what an array written among the kids holds, which the walk does not read —, and a trailer's `/Root` that names nothing `file.root-invalid`'s. | Every object reachable from the trailer, once each | -| `object.endobj-missing` | Warning | An object reachable from the trailer does not end with `endobj` (7.3.10): another token, or the end of the file, follows its value. The reader reads the value as far as it goes, as qpdf does while reporting it. An empty object, `2 0 obj endobj`, has its `endobj`. An object whose array or dictionary was never closed lacks it where the next object's header — `N G obj` in digits, white space alone between them, an object number from 1 to 2,147,483,647 and a generation up to 65,535 —, met where an element, a key or a value is expected, ends every container left open ([#230]); an array or a dictionary that holds such a header and is closed after it ends there all the same, its `endobj` judged missing, which none of the corpus's 344,361 objects does. After a stream whose data runs past the 8 KB window the reader first reads through, the file is asked for what follows its `endstream` ([#55]), read through white space and comments however long ([#174]); after an `endstream` inside that window, white space that runs past the window's edge leaves what follows unseen, and not judged. A stream with no `endstream` has its `endobj` when the object's `endobj` follows the length it declared, or ends its data with nothing but white space after it before the next object or the end of the file; one whose data runs on to the next object, or to the end of the file, lacks it. Where the object ends is unknown, and is not judged, after a stream whose declared length the reader kept with neither an `endstream` nor an `endobj` after it, one whose data, run on to the next object or the end of the file, holds an `endobj` that may be its object's own, and one whose end lies past what the reader reads first once the document's searches for `endstream` have read as much of the file as they may. | What follows each reachable object's value, as the reader read it | +| `object.endobj-missing` | Warning | An object reachable from the trailer does not end with `endobj` (7.3.10): another token, or the end of the file, follows its value. The reader reads the value as far as it goes, as qpdf does while reporting it. An empty object, `2 0 obj endobj`, has its `endobj`. An object whose array or dictionary was never closed lacks it where the next object's header — `N G obj` in digits, white space alone between them, an object number from 1 to 2,147,483,647 and a generation up to 65,535 —, met where an element, a key or a value is expected, ends every container left open ([#230]); an array or a dictionary that holds such a header and is closed after it ends there all the same, its `endobj` judged missing, which none of the corpus's 344,361 objects does. What follows a stream's `endstream` is read through white space and comments however long, and the file asked for it when it lies past the 8 KB window the reader first reads through — after a stream whose data runs past that window ([#55]), or an `endstream` the window's edge comes too soon after ([#174]). A stream with no `endstream` has its `endobj` when the object's `endobj` follows the length it declared — after white space and comments, a comment only once white space comes before it —, or ends its data with nothing but white space and comments after it before the next object or the end of the file; one whose data runs on to the next object, or to the end of the file, lacks it. Where the object ends is unknown, and is not judged, after a stream whose declared length the reader kept with neither an `endstream` nor an `endobj` after it, one whose data, run on to the next object or the end of the file, holds an `endobj` that may be its object's own, and one whose end lies past what the reader reads first once the document's searches for `endstream` have read as much of the file as they may. | What follows each reachable object's value, as the reader read it | | `object.name-null-character` | Warning | An object reachable from the trailer holds a name with a null character, written `#00`, which a name cannot contain (7.3.5). The reader keeps the name; qpdf refuses it and reads dictionaries without the keys so named. One finding per object, naming the first. | Every name of every reachable object | | `object.key-missing` | Warning | An object reachable from the trailer lacks a key the [Arlington PDF Model](#object-shapes-from-the-arlington-pdf-model) requires of its type — absent, or `null`, which in a dictionary is the same (7.3.7): a key the model requires once the version the file declares has it, or one it requires in some versions only, such as a form XObject's `/Name` in PDF 1.0. An inheritable key is looked for up the `/Parent` chain, and a field's `/DA` in the interactive form's too; a key only an extension of ISO 32000 defines is never required. An array shorter than the elements it requires lacks each required element it does not hold, one finding per element. [One finding per type and key](#object-shapes-from-the-arlington-pdf-model), at the first object, with how many lack it. | Every object reachable from the trailer, typed along the model's links, once each | | `object.value-type-wrong` | Warning | A value is of a type the Arlington PDF Model does not allow for its key, judged by the value's class: a real, even `400.0`, where an integer belongs; a string where a name does; a stream where a dictionary does, or the reverse, which is reported on the object that holds it and not walked into. A `null` written in an array is judged like any other value; one reached through a reference is not judged. One finding per type and key. | As `object.key-missing` | @@ -365,6 +365,7 @@ PDF/UA profiles of the `AdCodicem.Pdf.Conformance` package (M20) take families o [#219]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/219 [#220]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/220 [#221]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/221 +[#224]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/224 [#230]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/230 [#237]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/237 [#256]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/256 From 43e321485e0f22c83cedf66eb23c4694f129eae7 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 10:32:52 +0000 Subject: [PATCH 033/146] test: reach the two paths two coverage tests meant to and missed The test of a stream read again after the cache let it go kept the stream in the cache, which holds 64 objects at least: it now reads 70 others between its two readings, as the reader's check made for the first is meant to serve the second. The test of a stream parsed alone with no endstream in its window gave it a length, which settled it before any search: it now gives none. Refs #174 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- tests/AdCodicem.Pdf.Tests/ParserTests.cs | 6 +++--- tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs | 17 ++++++++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/AdCodicem.Pdf.Tests/ParserTests.cs b/tests/AdCodicem.Pdf.Tests/ParserTests.cs index b9a0dad4..f8b176a1 100644 --- a/tests/AdCodicem.Pdf.Tests/ParserTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ParserTests.cs @@ -200,9 +200,9 @@ public void Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_an [Fact] public void Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one() { - // Parsed alone in a window that is not the end of the data, a stream whose endstream and next header lie past it - // asks for a larger window, which may show them. - var parser = new PdfObjectParser(Encoding.ASCII.GetBytes("4 0 obj\n<< /Length 200 >>\nstream\nabc"), endsData: false); + // Parsed alone in a window that is not the end of the data, a stream with no length to confirm, whose endstream and + // next header lie past the window, asks for a larger one, which may show them. + var parser = new PdfObjectParser(Encoding.ASCII.GetBytes("4 0 obj\n<< >>\nstream\nabc"), endsData: false); parser.TryReadIndirectObject(out _, out _); diff --git a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs index 6c55bf88..731ff678 100644 --- a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs +++ b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs @@ -2239,11 +2239,22 @@ public void A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_ch file.Raw(Before.Replace("0000", Invariant($"{length:D4}"), StringComparison.Ordinal) + new string('a', length) + "\n% " + new string('c', 200) + "\nendstream\nendobj\n"); rows[6] = file.Object(6, "(six)"); - file.Table(7, rows, "/Size 7 /Root 1 0 R"); - using var document = PdfDocument.Open(file.Bytes, PdfReaderOptions.Default with { ObjectCacheCapacity = 1 }); + + for (var number = 10; number < 80; number++) + { + rows[number] = file.Object(number, Invariant($"({number})")); + } + + file.Table(80, rows, "/Size 80 /Root 1 0 R"); + using var document = PdfDocument.Open(file.Bytes, PdfReaderOptions.Default with { ObjectCacheCapacity = 64 }); document.GetObject(new PdfObjectId(4)).AsStream().Required().RawLength.Should().Be(length); - document.GetObject(new PdfObjectId(6)); + + for (var number = 10; number < 80; number++) + { + document.GetObject(new PdfObjectId(number)); + } + document.GetObject(new PdfObjectId(4)).AsStream().Required().RawLength.Should().Be(length); document.Diagnostics.Should().BeEmpty(); document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); From 7071b243675f221c99335c7aec459f613385cb21 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 10:33:07 +0000 Subject: [PATCH 034/146] docs: say how a deferred length's chain and its endobj are read, past the data a guard cut Lazy reading, the object-model reference and the validation rules say that data MaxStreamLength cut is not checked against a deferred /Length, that a /Length leading through references to an object no section read before places makes the section malformed, and that the endobj after a stream with no endstream is followed through the search's own reads. Refs #174, #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/website/docs/concepts/lazy-reading.md | 19 ++++++++++++------- docs/website/docs/reference/object-model.md | 5 +++-- .../docs/reference/validation-rules.md | 2 +- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/docs/website/docs/concepts/lazy-reading.md b/docs/website/docs/concepts/lazy-reading.md index e1e29fbe..d9030a9e 100644 --- a/docs/website/docs/concepts/lazy-reading.md +++ b/docs/website/docs/concepts/lazy-reading.md @@ -19,7 +19,7 @@ Opening a document builds an index, finds the catalog, and stops there. the map the chain gives is the file's own. A `/Length` no section read before places, or that leads through references to such an object, is read once the chain is, and checked against the data the chain took up to the first `endstream` after the data's start, searched for in the file up to the next object the sections read so far - place, or its end; one that leads to an object `MaxObjectLength` cuts waits for the chain the same way, and is the + place, or its end — unless `MaxStreamLength` cut that data, which is the guard's; one that leads to an object `MaxObjectLength` cuts waits for the chain the same way, and is the guard's: the guard is reached once the chain is read, and the length is not checked. Either way, when the section's own row for that `/Length` — the row the reading of its rows keeps — lies among the rows the data holds, those rows spelled `endstream`: the object is read where the row places it, without being kept, and the data is taken @@ -71,7 +71,10 @@ the chain is read: - when an object the map lacks is asked for, by opening or after it, while the map may have lost entries: a cross-reference section the chain names cannot be found, or a `/Prev` or `/XRefStm` is not an offset — one written as a reference no section read before it places included — (`xref.section-missing`); one is found and - cannot be read (`xref.section-unreadable`), a value its rows need written as such a reference, or a `/Length` so written whose data no `endstream` ends before the next object or the end of the file, among the causes; the chain loops back on itself (`xref.chain-cycle`); a row of a classic table cannot be read, which ends its subsection; a row gives an object + cannot be read (`xref.section-unreadable`), a value its rows need written as such a reference, or a `/Length` so + written, or that leads through references to an object no section read before places, whose data no `endstream` ends + before the next object or the end of the file, among the causes; the chain loops back on itself (`xref.chain-cycle`); + a row of a classic table cannot be read, which ends its subsection; a row gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, a field wider than 8 bytes whose leading bytes are not all zero —, which refuses that row alone, no older section's row standing for @@ -135,11 +138,13 @@ past the window, it asks the file for the bytes after the declared length rather as far as white space and comments run. A length nothing confirms, or a `/Length` that gives none, sends the reader on one search for the first `endstream` after the data's start: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold and the next object the index as the file wrote it places, or the end -of the file. With no `endstream` before that stop, the object's `endobj` plays its part. The search is made once for -each stream, however often the stream is parsed again; no window grows to find an `endstream`, so that a stream's data -never counts toward `MaxObjectLength`; and the searches of a document read four times the file at most — a valid -file's, about twice, for the white space after a declared length and the cross-reference streams whose `/Length` the -chain cannot read yet. What each case keeps, and reports, is in [Diagnostics](diagnostics.md#where-a-stream-ends). +of the file. With no `endstream` before that stop, the object's `endobj` plays its part: the search follows the last one +through its own reads, the comments after it taken for white space, rather than read the bytes before the stop again +backward. The search is made once for each stream, however often the stream is parsed again, and whether the object's +`endobj` follows the declared length is read again only when a rebuild changes that length; no window grows to find an +`endstream`, so that a stream's data never counts toward `MaxObjectLength`; and the searches of a document read four +times the file at most — a valid file's, about twice, for the white space after a declared length and the +cross-reference streams whose `/Length` the chain cannot read yet. What each case keeps, and reports, is in [Diagnostics](diagnostics.md#where-a-stream-ends). When it is read or decoded, a stream works under the [limits](reader-limits.md) of the document it came from, however long after opening — so with `ThrowOnLimit` set, `GetRawBytes()` and `Decode()` can throw long after `Open` diff --git a/docs/website/docs/reference/object-model.md b/docs/website/docs/reference/object-model.md index db3625a1..7d5c562b 100644 --- a/docs/website/docs/reference/object-model.md +++ b/docs/website/docs/reference/object-model.md @@ -102,8 +102,9 @@ read or decoded, and for as long as the caller keeps what that gave. Opened with `ThrowOnLimit`, a document's streams throw a `PdfLimitExceededException` from `GetRawBytes()` and `Decode()` instead, each time their data runs past `MaxStreamLength`; `Decode()` throws at the other limits it reaches -too, and `HasImageFilter()` when loading a value the filters read reaches one. No call's result is kept for the next: a caller that uses the data twice keeps what one call gave, rather than call -again ([Tune the reader's memory](../guides/tune-reader-memory.md#read-a-streams-data-once)). +too, and `HasImageFilter()` when loading a value the filters read reaches one. No call's result is kept for the next: +a caller that uses the data twice keeps what one call gave, rather than call again +([Tune the reader's memory](../guides/tune-reader-memory.md#read-a-streams-data-once)). Data built with `PdfStreamData.FromMemory` is held by the stream, whatever its length, and handed back as it was given; its `Position` is -1. diff --git a/docs/website/docs/reference/validation-rules.md b/docs/website/docs/reference/validation-rules.md index c7f888f0..706fa1e8 100644 --- a/docs/website/docs/reference/validation-rules.md +++ b/docs/website/docs/reference/validation-rules.md @@ -51,7 +51,7 @@ References are to ISO 32000-1:2008. | Rule | Severity | Meaning | Reads | |---|---|---|---| -| `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]); or a cross-reference stream whose `/Length` is such a reference, or leads through references to an object no section read before it places, and whose data no `endstream` ends before the next object the sections read so far place, or the end of the file — or was not searched for one, the document's searches for `endstream` having read as much of the file as they may ([#220]). Rows that spell `endstream` end such a stream's data there, and the section lacks the rows past them, unless the section's own row for that `/Length` — the row the reading of its rows keeps, one of a reserved type ignored —, among the rows before them, places an object that gives a longer length an `endstream` confirms: the data is then taken to it. A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. Nor is a cross-reference stream whose value the rows are read with leads to an object a limit cut: the section counts as cut there, the chain ends at it, nothing is rebuilt, and `xref.checked-in-part` says so ([#263]). Nor is one whose `/Length`, which the chain could not read, leads to an object `MaxObjectLength` cuts, where only that length could say where the data ends: with no `endstream` before the next object or the end of the file, the section counts as cut, as above; when its rows spell one, the integer the cut left of the object the section's own row places is where an `endstream` is looked for, the data taken to it when one confirms it, and the rows past the spelled one the limit's otherwise ([#224]). | Each section of the chain | +| `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]); or a cross-reference stream whose `/Length` is such a reference, or leads through references to an object no section read before it places, and whose data no `endstream` ends before the next object the sections read so far place, or the end of the file — or was not searched for one, the document's searches for `endstream` having read as much of the file as they may ([#220]). Rows that spell `endstream` end such a stream's data there, and the section lacks the rows past them, unless the section's own row for that `/Length` — the row the reading of its rows keeps, one of a reserved type ignored —, among the rows before them, places an object that gives a longer length an `endstream` confirms: the data is then taken to it. A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. Nor is a cross-reference stream whose value the rows are read with leads to an object a limit cut: the section counts as cut there, the chain ends at it, nothing is rebuilt, and `xref.checked-in-part` says so ([#263]). Nor is one whose `/Length`, which the chain could not read, leads to an object `MaxObjectLength` cuts, where only that length could say where the data ends: with no `endstream` before the next object or the end of the file, the section counts as cut, as above; when its rows spell one, the integer the cut left of the object the section's own row places is where an `endstream` is looked for, the data taken to it when one confirms it, and otherwise the rows past the spelled one the limit's, the section counting as cut there, as above ([#224]). | Each section of the chain | | `xref.section-not-found` | Error | A section a `/Prev` or an `/XRefStm` names is neither there nor within 512 bytes of it, or what names it is not an offset — tiff2pdf writes `/Prev 576066 0 R`. The objects only it indexes are missing from the index. The section `startxref` names is `file.startxref-wrong`'s. A `/Prev` or an `/XRefStm` written as a reference that leads to an object `MaxObjectLength` cuts names no section missing: the chain ends there for the limit, the sections older than it unread ([#263]). | Each section of the chain | | `xref.section-shifted` | Warning | A section a `/Prev` or an `/XRefStm` names starts a few bytes from where it is named, and the reader found it there. | Each section of the chain | | `xref.chain-loop` | Error | The chain names a section it has already read, or an offset it has already named where it read none, in the file or outside it: the reader stops there, and the section the chain should have gone on to is unknown, what only it indexed found, if at all, by rebuilding the index — as for `xref.section-not-found`. | The chain | From 0e22445f529e9b066e270c42d36b183d8f0d1dc4 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 10:44:20 +0000 Subject: [PATCH 035/146] fix: say what a /Prev or /XRefStm a guard cut leaves unchecked A /Prev or an /XRefStm that leads to an object MaxObjectLength cuts ends the chain for the guard, and the validation said nothing of the older sections it hides: xref.checked-in-part was silent, and file.size-wrong and page-tree.page-orphaned judged the index as whole. The section keeps what the entry is, the fault naming the reference cut, as ChainHiddenByGuard, copied when it is relocated; xref.checked-in-part says at the section that the sections it names were not checked, and the two rules that need a whole chain leave it unjudged. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../docs/reference/validation-rules.md | 8 +++---- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 22 ++++++++++++------- .../IO/XRef/XRefSectionRecord.cs | 8 +++++++ .../Validation/Rules/CheckedInPartRule.cs | 11 ++++++++++ .../Rules/PageTreePageOrphanedRule.cs | 3 ++- .../Validation/Rules/SizeWrongRule.cs | 3 ++- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 15 +++++++++---- 7 files changed, 52 insertions(+), 18 deletions(-) diff --git a/docs/website/docs/reference/validation-rules.md b/docs/website/docs/reference/validation-rules.md index 706fa1e8..e661ea76 100644 --- a/docs/website/docs/reference/validation-rules.md +++ b/docs/website/docs/reference/validation-rules.md @@ -44,7 +44,7 @@ References are to ISO 32000-1:2008. | `file.trailer-missing` | Error | A cross-reference table's rows are not followed by the `trailer` keyword: they run into a dictionary, or the file ends. qpdf, as the reader, then rebuilds the index. | Each classic table of the chain | | `file.trailer-malformed` | Error | A trailer is not a well-formed dictionary: the `trailer` keyword is not followed by one, or the reader read it despite syntax errors — a key that is not a name, a dictionary or a string left unclosed, a hexadecimal string holding bytes that are neither digits nor white space, a name whose number sign is no escape, a key given twice —, so an entry may have been lost with the syntax, or read otherwise than its writer meant. A cross-reference stream's dictionary is held to the same. | Each trailer of the chain, as the reader parsed it | | `file.root-invalid` | Error | The trailer's `/Root` is missing or does not lead to a document catalog (Table 15), a value that is not a reference included — unless it is a catalog written in the trailer itself, which the reader takes as it is ([#111]). The reader then looks for an object of `/Type /Catalog` among those the index holds — and rebuilds the index only when none is one —: the catalog it finds is its choice, not the file's. Not judged when the trailer could not be read at all, nor when the index was rebuilt as the document opened, nor when one of the reader's limits left the catalog out of the index as the document opened, which `xref.checked-in-part` says, or cut the object `/Root` names — through a chain of references too, and in an object stream whose cut leaves it unread, past the cut data or in a stream the guard made unreadable ([#259]), or in one whose filters end at a `/Filter`, a `/DecodeParms` or a parameter that leads to an object a guard cut or left out of the index, which leaves its data encoded ([#263]) — or the trailer a catalog written there was read from, which the reader then takes for the catalog as it read it, looking for no other, and its `limit.*` report names ([#175]) — whatever a rebuild found since —, nor a `/Root` the trailers lack where one of the reader's limits kept a trailer from being read whole — cut, or older than where the chain stopped, at the section count or at a section whose rows a limit cut or filled the index with —: it may lie there, and the reader rebuilds nothing for it. Rows a limit cut that run on to no trailer lack one: the file's, which a rebuild answers. The catalog the finding names is the one the reader took last, in the rebuild run before the first rule if there was one. | The trailer the chain gave, merged as readers merge it, and `/Root` before the reader looked for a catalog | -| `file.size-wrong` | Warning | A section's `/Size` is missing, or is not one more than the highest object number it and the sections it updates use (Tables 15 and 17, 7.5.6, 7.5.8.4). A hybrid file's cross-reference stream meets one of Table 17's two readings or the other — one more than the highest number it and the sections it updates use, or the `/Size` of the table that names it —, and either is accepted. A `/Size` that leaves in-use objects numbered above it is `xref.object-past-size`'s. A `/Size` written as a reference, or as a real, is said to be. Not judged when the chain lost a section, or one of the reader's limits cut it. | Each section's trailer, and the highest number each section indexes | +| `file.size-wrong` | Warning | A section's `/Size` is missing, or is not one more than the highest object number it and the sections it updates use (Tables 15 and 17, 7.5.6, 7.5.8.4). A hybrid file's cross-reference stream meets one of Table 17's two readings or the other — one more than the highest number it and the sections it updates use, or the `/Size` of the table that names it —, and either is accepted. A `/Size` that leaves in-use objects numbered above it is `xref.object-past-size`'s. A `/Size` written as a reference, or as a real, is said to be. Not judged when the chain lost a section, or one of the reader's limits cut it or ended it before its older sections ([#263]). | Each section's trailer, and the highest number each section indexes | | `file.trailer-value-wrong` | Warning | A value a section's trailer gives the reader to read the chain is written as the specification does not allow, and read all the same, as other readers read it: a reference where it must be direct, read where a section read before it places the object; a real with no fractional part where it must be an integer, read as the integer it equals ([#182], [#215]). A cross-reference stream's `/Type`, `/Index`, `/Prev` and `/W`, the elements of `/Index` and `/W`, and its `/Filter` and `/DecodeParms` are direct (7.5.8.2); in a file that declares PDF 2.0, so are its `/Length` and the elements and entries of its `/Filter` and `/DecodeParms` (an erratum of ISO 32000-2, pdf-issues #246). A classic trailer's `/Prev` is direct (Table 15); an `/XRefStm` may be indirect (Table 19). `/W`, `/Index`, `/Prev`, `/XRefStm`, a stream's `/Length` and its `/Predictor`, `/Colors`, `/BitsPerComponent`, `/Columns` and `/EarlyChange` are integers. One finding per value, which says both faults of a reference that leads to a real. A value the chain is read with that the reader could not read so is the section's fault — `xref.section-malformed`, `xref.section-not-found` —, unless it leads to an object one of the reader's limits cut, where the chain ends for the limit, a section whose rows need it left to `xref.checked-in-part` ([#263]), and `/Size` is `file.size-wrong`'s; a `/Type`, or a `/Length` read once the chain is, is reported here whether or not it could be read. | Each section's trailer, as the file wrote it, and the objects its references name, through the index the reader settled on before the first rule | ### Cross-references @@ -52,7 +52,7 @@ References are to ISO 32000-1:2008. | Rule | Severity | Meaning | Reads | |---|---|---|---| | `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]); or a cross-reference stream whose `/Length` is such a reference, or leads through references to an object no section read before it places, and whose data no `endstream` ends before the next object the sections read so far place, or the end of the file — or was not searched for one, the document's searches for `endstream` having read as much of the file as they may ([#220]). Rows that spell `endstream` end such a stream's data there, and the section lacks the rows past them, unless the section's own row for that `/Length` — the row the reading of its rows keeps, one of a reserved type ignored —, among the rows before them, places an object that gives a longer length an `endstream` confirms: the data is then taken to it. A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. Nor is a cross-reference stream whose value the rows are read with leads to an object a limit cut: the section counts as cut there, the chain ends at it, nothing is rebuilt, and `xref.checked-in-part` says so ([#263]). Nor is one whose `/Length`, which the chain could not read, leads to an object `MaxObjectLength` cuts, where only that length could say where the data ends: with no `endstream` before the next object or the end of the file, the section counts as cut, as above; when its rows spell one, the integer the cut left of the object the section's own row places is where an `endstream` is looked for, the data taken to it when one confirms it, and otherwise the rows past the spelled one the limit's, the section counting as cut there, as above ([#224]). | Each section of the chain | -| `xref.section-not-found` | Error | A section a `/Prev` or an `/XRefStm` names is neither there nor within 512 bytes of it, or what names it is not an offset — tiff2pdf writes `/Prev 576066 0 R`. The objects only it indexes are missing from the index. The section `startxref` names is `file.startxref-wrong`'s. A `/Prev` or an `/XRefStm` written as a reference that leads to an object `MaxObjectLength` cuts names no section missing: the chain ends there for the limit, the sections older than it unread ([#263]). | Each section of the chain | +| `xref.section-not-found` | Error | A section a `/Prev` or an `/XRefStm` names is neither there nor within 512 bytes of it, or what names it is not an offset — tiff2pdf writes `/Prev 576066 0 R`. The objects only it indexes are missing from the index. The section `startxref` names is `file.startxref-wrong`'s. A `/Prev` or an `/XRefStm` written as a reference that leads to an object `MaxObjectLength` cuts names no section missing: the chain ends there for the limit, the sections older than it unread, which `xref.checked-in-part` says ([#263]). | Each section of the chain | | `xref.section-shifted` | Warning | A section a `/Prev` or an `/XRefStm` names starts a few bytes from where it is named, and the reader found it there. | Each section of the chain | | `xref.chain-loop` | Error | The chain names a section it has already read, or an offset it has already named where it read none, in the file or outside it: the reader stops there, and the section the chain should have gone on to is unknown, what only it indexed found, if at all, by rebuilding the index — as for `xref.section-not-found`. | The chain | | `xref.entry-broken` | Error | An in-use entry places its object neither at its offset nor within 512 bytes of it, in an object stream whose header does not list it, or in one where it is written as a stream or holds one, which an object stream cannot hold (7.5.7), and which reads as null — the index the reader rebuilt included, where it places such an object and the file's own index places nothing, or the part of it the reader read where one of its limits stopped the chain short ([#161]) — as a load decides it: the header read at the offset after white space and comments, however long, a whole token no header begins with ruling one out at once, and the one found near it confirmed by what a load parses there ([#175], [#256]). An offset outside the file — past its end, before its start, or past what a `long` holds once the header's offset is added — is looked for nearby as a load looks: an object found there is `xref.entry-shifted`'s. One finding per entry, located at the object and the offset the entry gives, or at the object alone when that offset lies outside the file. | Each object's header, as a load reads it, a few dozen bytes in a sound file; each object stream's header, and each member whose bytes hold `stream`, parsed as a load parses it | @@ -63,7 +63,7 @@ References are to ISO 32000-1:2008. | `xref.object-stream-value-wrong` | Warning | An object stream gives its `/N`, `/First` or `/Length` as a real number with no fractional part, where the specification asks for an integer (Tables 5 and 16). The reader reads it as the integer it equals, as MuPDF, PDFBox and pdf.js do; qpdf and poppler lose the stream's objects ([#215]). One finding per stream and key. A value with a fractional part is another fault: a `/N` or a `/First` leaves the stream unreadable, `xref.object-stream-broken`'s, and a `/Length` is recovered through the stream's `endstream`, which the reader reports (`stream.length-invalid`). | As `xref.object-stream-circular` | | `xref.offset-imprecise` | Warning | Offsets name the white space before what they designate rather than its first byte (7.5.4, 7.5.5): Microsoft Print to PDF names the line feed before each object, and some tools the one before `xref`. Every reader skips the white space. One finding for the file, with how many offsets and the first of them. | The sections of the chain, and each object's header | | `xref.object-past-size` | Warning | In-use objects are numbered above the trailer's `/Size`, which Table 15 makes a conforming reader ignore ("shall be ignored and defined to be missing"). The entries, the objects' headers and the references agree against `/Size`, and the reader reads those objects, as qpdf does while warning. One finding for the file, with how many objects and the first of them; `file.size-wrong` then says nothing more of that `/Size`. | The index and the trailer's `/Size` | -| `xref.checked-in-part` | Information | Part of the index was not checked: one of the reader's limits stopped it reading a section, a trailer or an object stream whole, the chain past `MaxXRefSectionCount` sections, or the index past the `MaxIndexEntries` object numbers it holds, said once, where it filled — raising the limit lets the rest be checked —; an entry whose header, where it places its object or near it, runs past `MaxObjectLength`; the entries the probe left undecided once reading their headers had consumed twice the file's length — the bytes crossed up to each decision, past the first 64 of each, and every byte read past a run an entry skips, which a valid file's headers, distinct bytes, never reach —, said once ([#175]); or the document is encrypted and its object streams are readable only once decrypted (M16). A chain cut where it goes on outside the file is located at the section whose `/Prev` names where. | What the reader and the probe of the index recorded | +| `xref.checked-in-part` | Information | Part of the index was not checked: one of the reader's limits stopped it reading a section, a trailer or an object stream whole, the chain past `MaxXRefSectionCount` sections, or the index past the `MaxIndexEntries` object numbers it holds, said once, where it filled — raising the limit lets the rest be checked —; an entry whose header, where it places its object or near it, runs past `MaxObjectLength`; the entries the probe left undecided once reading their headers had consumed twice the file's length — the bytes crossed up to each decision, past the first 64 of each, and every byte read past a run an entry skips, which a valid file's headers, distinct bytes, never reach —, said once ([#175]); the sections older than one whose `/Prev` or `/XRefStm` leads to an object `MaxObjectLength` cuts, which end the chain there, said at that section with the reference that names them ([#263]); or the document is encrypted and its object streams are readable only once decrypted (M16). A chain cut where it goes on outside the file is located at the section whose `/Prev` names where. | What the reader and the probe of the index recorded | The entries the `xref.entry-*`, `xref.generation-mismatch`, `xref.object-stream-*`, `xref.offset-imprecise` and `xref.object-past-size` rules check are those of the index the file's chain gave, as the reader first read @@ -110,7 +110,7 @@ object nothing reachable refers to is not judged by them. | `page-tree.parent-wrong` | Warning | A node or a page has no `/Parent`, or one that is not the node listing it; or the root has one (Tables 29 and 30). The reader, as qpdf, walks the tree through `/Kids` and inherits along that path. | The page tree | | `page-tree.mediabox-invalid` | Warning | A page has no `/MediaBox`, its own or inherited, or one that is not four numbers enclosing an area (Table 30, 7.9.5): its size is unknown, and qpdf gives it a US Letter sheet's. Any two opposite corners make a rectangle, so a box written from its upper corner is sound. Missing, reported for each page; malformed, where it is written. | The page tree, and each media box | | `page-tree.resources-missing` | Warning | A page has no `/Resources`, its own or inherited (Table 30), where an empty dictionary says it uses none. | The page tree | -| `page-tree.page-orphaned` | Information | An object of `/Type /Page`, with a `/Parent` or a `/Contents`, that the page tree does not list: no reader shows it. Nothing forbids one — an edit leaves them —; a marked-content property list typed `/Page`, as DocuSign writes, is not a page. Judged only where the file's own index is sound and whole and the tree was read whole — no part of it left unread or cut by one of the reader's limits —, since elsewhere loading an object may bring back what an update deleted, and a page the tree seems not to list may lie past a cut. | Every object the file's index holds in use, once each, without its content | +| `page-tree.page-orphaned` | Information | An object of `/Type /Page`, with a `/Parent` or a `/Contents`, that the page tree does not list: no reader shows it. Nothing forbids one — an edit leaves them —; a marked-content property list typed `/Page`, as DocuSign writes, is not a page. Judged only where the file's own index is sound and whole — no section lost, cut, or whose older sections one of the reader's limits hid ([#263]) — and the tree was read whole — no part of it left unread or cut by one of the reader's limits —, since elsewhere loading an object may bring back what an update deleted, and a page the tree seems not to list may lie past a cut. | Every object the file's index holds in use, once each, without its content | The page tree is walked once from the catalog's `/Pages`, through `/Kids`, as qpdf walks it: a node is a dictionary with a `/Kids` array or of `/Type /Pages`, any other dictionary the tree lists is a page, and pages are counted as the diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 36c8f93d..0390a60b 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3027,12 +3027,13 @@ private bool TryReadClassicTable(long absolute, XRefSectionRecord section, out l if (ReadTrailer(window.Memory, absolute, lexer.Position, windowFull, section) is { } trailer) { MergeSectionTrailer(trailer, section); - previous = SectionOffset(trailer, PdfName.Prev, absolute, out _); + previous = SectionOffset(trailer, PdfName.Prev, absolute, out var previousHidden); hybrid = SectionOffset(trailer, PdfName.XRefStm, absolute, out var streamHidden); + section.ChainHiddenByGuard = streamHidden ?? previousHidden; // A stream a guard hid holds rows newer than the older sections': the chain ends with it, as with a // stream a guard cut, since their rows would stand for the stream's. - previous = streamHidden ? -1 : previous; + previous = streamHidden is null ? previous : -1; } return true; @@ -3787,7 +3788,9 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect _rowsLeftUnread = RowsFrom(index is null ? [0, declared] : subsections, stopped, rowsCut ? long.MaxValue : held); _xref.MergeTrailer(dictionary); - previous = rowsCut ? -1 : SectionOffset(dictionary, PdfName.Prev, absolute, out _); + string? previousHidden = null; + previous = rowsCut ? -1 : SectionOffset(dictionary, PdfName.Prev, absolute, out previousHidden); + section.ChainHiddenByGuard = previousHidden; return XRefSectionState.Read; } @@ -4103,10 +4106,13 @@ public bool TryRead(PdfName key, PdfObject? written, bool entry, out PdfObject? /// The section's trailer, or its stream's dictionary. /// The entry to read, /Prev or /XRefStm. /// The offset of the section, which a report names. - /// Whether the entry leads to an object a guard cut, which ends the chain. - private long SectionOffset(PdfDictionary trailer, PdfName key, long section, out bool hidden) + /// + /// What the entry is, when it leads to an object a guard cut, which ends the chain — "its /Prev is the reference 9 0 R, + /// which one of the reader's limits cut" —; null otherwise. + /// + private long SectionOffset(PdfDictionary trailer, PdfName key, long section, out string? hidden) { - hidden = false; + hidden = null; var value = trailer.GetRaw(key); @@ -4131,12 +4137,12 @@ private long SectionOffset(PdfDictionary trailer, PdfName key, long section, out // So does one leading to an object a guard cut, whose loading, once the chain is read, reaches the guard (#263). The // older sections, and their trailers, are the guard's to hide. - if (value is PdfReference && Follow(key, value, entry: true, out _, out var cut) is not null && cut is { } past) + if (value is PdfReference && Follow(key, value, entry: true, out _, out var cut) is { } fault && cut is { } past) { _cutByGuard = true; _structure.TrailersHiddenByGuard = true; (_chainValuesCut ??= []).Add(past); - hidden = true; + hidden = fault; return -1; } diff --git a/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs b/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs index 75077127..5913ca53 100644 --- a/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs +++ b/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs @@ -73,6 +73,13 @@ public XRefSectionRecord(string namedBy, long namedOffset, long namedFrom) /// public bool TrailerCutByLimit { get; set; } + /// + /// Gets what the section's /Prev or /XRefStm is, when it leads to an object one of the reader's guards cut + /// — "its /Prev is the reference 9 0 R, which one of the reader's limits cut" —, which ends the chain here: the sections + /// it names are the guard's to hide (#263). Null when nothing did. + /// + public string? ChainHiddenByGuard { get; set; } + /// Gets the section's own trailer — a cross-reference stream's dictionary —, or null when none was read. public PdfDictionary? Trailer { get; set; } @@ -118,6 +125,7 @@ public void RelocateTo(XRefSectionRecord attempt) Incomplete = attempt.Incomplete; CutByLimit = attempt.CutByLimit; TrailerCutByLimit = attempt.TrailerCutByLimit; + ChainHiddenByGuard = attempt.ChainHiddenByGuard; Trailer = attempt.Trailer; TrailerFault = attempt.TrailerFault; TrailerPosition = attempt.TrailerPosition; diff --git a/src/AdCodicem.Pdf/Validation/Rules/CheckedInPartRule.cs b/src/AdCodicem.Pdf/Validation/Rules/CheckedInPartRule.cs index 066b7919..e95cbb8d 100644 --- a/src/AdCodicem.Pdf/Validation/Rules/CheckedInPartRule.cs +++ b/src/AdCodicem.Pdf/Validation/Rules/CheckedInPartRule.cs @@ -71,6 +71,17 @@ public void Check(ValidationContext context) $"One of the reader's limits stopped it reading the cross-reference section at offset {section.Offset} whole: what lies past the limit was not checked."), remedy: null); } + + if (section.ChainHiddenByGuard is { } hidden) + { + context.Report( + this, + PdfValidationLocation.AtPosition(section.Offset), + string.Create( + CultureInfo.InvariantCulture, + $"The cross-reference section at offset {section.Offset} names older sections, but {hidden}: they were not checked."), + remedy: null); + } } foreach (var finding in context.Probe.NotChecked) diff --git a/src/AdCodicem.Pdf/Validation/Rules/PageTreePageOrphanedRule.cs b/src/AdCodicem.Pdf/Validation/Rules/PageTreePageOrphanedRule.cs index f7afb5c4..da2a2acc 100644 --- a/src/AdCodicem.Pdf/Validation/Rules/PageTreePageOrphanedRule.cs +++ b/src/AdCodicem.Pdf/Validation/Rules/PageTreePageOrphanedRule.cs @@ -92,7 +92,8 @@ private static bool IsIndexSound(ValidationContext context) foreach (var section in structure.Sections) { - if (section.State is not (XRefSectionState.Read or XRefSectionState.Relocated) || section.Incomplete || section.CutByLimit) + if (section.State is not (XRefSectionState.Read or XRefSectionState.Relocated) || section.Incomplete || section.CutByLimit || + section.ChainHiddenByGuard is not null) { return false; } diff --git a/src/AdCodicem.Pdf/Validation/Rules/SizeWrongRule.cs b/src/AdCodicem.Pdf/Validation/Rules/SizeWrongRule.cs index cab66f1a..af207118 100644 --- a/src/AdCodicem.Pdf/Validation/Rules/SizeWrongRule.cs +++ b/src/AdCodicem.Pdf/Validation/Rules/SizeWrongRule.cs @@ -135,7 +135,8 @@ private static bool IsWhole(IReadOnlyList sections) { foreach (var section in sections) { - if (section.State is XRefSectionState.NotFound or XRefSectionState.Malformed || section.CutByLimit) + if (section.State is XRefSectionState.NotFound or XRefSectionState.Malformed || section.CutByLimit || + section.ChainHiddenByGuard is not null) { return false; } diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index c0a063ac..f58fceac 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -528,16 +528,23 @@ public void A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s(string ke { // The newer section's /Prev, object 9, or its /XRefStm, object 6, is one that section places and MaxObjectLength cuts // before its value: the section it names was said to be missing, and the index rebuilt. The chain ends there for the - // guard, which is reached once the chain is read — thrown at opening when the caller asks for it (#263). + // guard, which is reached once the chain is read — thrown at opening when the caller asks for it (#263). The + // validation says the sections it names were not checked, and judges nothing the chain they end would hold. + var reference = key == "Prev" ? "9 0 R" : "6 0 R"; var file = key == "Prev" - ? ChainFiles.UpdateNamingItsPrevious("9 0 R", padding: 3000) - : ChainFiles.HybridNamingItsStream("6 0 R", padding: 3000); + ? ChainFiles.UpdateNamingItsPrevious(reference, padding: 3000) + : ChainFiles.HybridNamingItsStream(reference, padding: 3000); using var document = PdfDocument.Open(file, ObjectGuard); document.WasRepaired.Should().BeFalse(); document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); document.Reader.Structure.Sections.Should().ContainSingle(); - new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionNotFound).Should().BeFalse(); + var report = new PdfValidator().Validate(document); + report.Contains(PdfValidationRuleIds.XRefSectionNotFound).Should().BeFalse(); + report.Contains(PdfValidationRuleIds.FileSizeWrong).Should().BeFalse(); + var section = document.Reader.Structure.Sections[0].Offset; + report.Findings.Where(finding => finding.RuleId == PdfValidationRuleIds.XRefCheckedInPart).Select(finding => finding.Message).Should().Equal( + $"The cross-reference section at offset {section} names older sections, but its /{key} is the reference {reference}, which one of the reader's limits cut: they were not checked."); var throwing = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); From 0131a801b49df763caea5d76868bf9496136292b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 10:50:28 +0000 Subject: [PATCH 036/146] fix: name the link a cross-reference stream's /Length chain stops at A cross-reference stream whose /Length the chain could not read, and whose data no endstream ends, was said to hold the first reference of the chain as one no section places, even where a newer section placed it and the chain stopped further on. The fault now gives what reading the chain without loading says, as the section's other values do: the link no section read so far places, "is" for the entry's own reference and "holds" for a later link. The two tests that quoted the first link's wording, and the threat model, now read "its /Length is the reference": the wording alone changed, not what is asserted. Refs #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 2 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 13 +++++++---- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 23 +++++++++++++++---- .../DocumentReaderTests.cs | 20 ++++++++++++++-- 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index 8db3d658..615d3f4e 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -283,7 +283,7 @@ Known gaps: | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s` | Yes | | What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length holds the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places, the integer read before the cut is only a place to look for `endstream` at: the data is taken again where one confirms it, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, and nothing is charged to the file ([#224]). A row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data `MaxStreamLength` cut is the guard's, and not checked against it ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places, the integer read before the cut is only a place to look for `endstream` at: the data is taken again where one confirms it, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, and nothing is charged to the file ([#224]). A row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data `MaxStreamLength` cut is the guard's, and not checked against it ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 0390a60b..fb8dcf0a 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3564,13 +3564,16 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // file, ends nowhere the section says: its rows are not read (#220). Where the /Length leads to an object a guard cut, // the length that would have said is the guard's, as a value its rows are read with is (#263): the section counts as // cut, the chain ends here, and the guard is reached once the chain is read. - if (deferred is { } unended && _endStreamSearches.TryGetValue((xrefStream.Data.Position, true), out var known) && + if (deferred is not null && _endStreamSearches.TryGetValue((xrefStream.Data.Position, true), out var known) && known.Search is { Length: null } search) { - if (Follow(PdfName.Length, dictionary.GetRaw(PdfName.Length), entry: true, out _, out var cutObject) is { } cutFault && - cutObject is { } past) + // The fault names the link of the chain that could not be read — the one no section read so far places, or the + // one a guard cut —, which the chain read without loading stopped at, as Follow does. + var lengthFault = Follow(PdfName.Length, dictionary.GetRaw(PdfName.Length), entry: true, out _, out var cutObject); + + if (cutObject is { } past) { - section.Fault = cutFault; + section.Fault = lengthFault; section.CutByLimit = true; (_chainValuesCut ??= []).Add(past); return XRefSectionState.Malformed; @@ -3581,7 +3584,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect : "before the end of the file"; section.Fault = string.Create( CultureInfo.InvariantCulture, - $"its /Length holds the reference {unended.Length.Number} {unended.Length.Generation} R, which no section read before it places where it can be read, and {(search.Searched ? "no endstream follows its data " + where : "its data was not searched for its endstream, the document's searches for endstream having read as much of the file as they may")}"); + $"{lengthFault}, and {(search.Searched ? "no endstream follows its data " + where : "its data was not searched for its endstream, the document's searches for endstream having read as much of the file as they may")}"); return XRefSectionState.Malformed; } diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index ca1d3c22..0becf2be 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -247,9 +247,14 @@ public static byte[] AloneWithIndirectLengthAndNoTypeField() /// endstream after an end-of-line when . With /// , stream 7's /Length names object 9, which only stream 7 indexes, and a comment /// sets the update further from it than a search for a section's misplaced offset reaches. Object 4 holds - /// , in which {length} stands for the length. The file is valid. + /// , in which {length} stands for the length. The file is valid. With + /// , the update writes object 9 again, holding it, and stream 10 indexes it there. /// - public static byte[] UpdateWithIndirectLength(bool spellingEndStream, bool olderIndirect = false, string four = "{length}") + public static byte[] UpdateWithIndirectLength( + bool spellingEndStream, + bool olderIndirect = false, + string four = "{length}", + string? nine = null) { using var file = new Writer("1.5"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); @@ -277,15 +282,23 @@ public static byte[] UpdateWithIndirectLength(bool spellingEndStream, bool older } file.Object(8, "(new)"); + + if (nine is not null) + { + file.Object(9, nine); + } + var spelled = spellingEndStream ? "\nendstream\0\0\0\0"u8.ToArray() : Encode([(0, 0, 0), (0, 0, 0)], Rows.Raw); - file.Object(4, four.Replace("{length}", (7 + spelled.Length + 14).ToString(CultureInfo.InvariantCulture), StringComparison.Ordinal)); + var after = nine is null ? 14 : 21; + file.Object(4, four.Replace("{length}", (7 + spelled.Length + after).ToString(CultureInfo.InvariantCulture), StringComparison.Ordinal)); var second = file.Position; var data = new List(Encode([(1, file.OffsetOf(4), 0)], Rows.Raw)); data.AddRange(spelled); - data.AddRange(Encode([(1, file.OffsetOf(8), 0), (1, second, 0)], Rows.Raw)); + data.AddRange(Encode(nine is null ? [(1, file.OffsetOf(8), 0), (1, second, 0)] : [(1, file.OffsetOf(8), 0), (1, file.OffsetOf(9), 0), (1, second, 0)], Rows.Raw)); + var index = nine is null ? "4 3 8 1 10 1" : "4 3 8 3"; file.Stream( 10, - string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size 11 /W [1 4 2] /Index [4 3 8 1 10 1] /Root 1 0 R /Prev {first} /Length 4 0 R"), + string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size 11 /W [1 4 2] /Index [{index}] /Root 1 0 R /Prev {first} /Length 4 0 R"), [.. data]); file.StartXRef(second); return file.ToArray(); diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index f2728814..013f75dc 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -2101,7 +2101,7 @@ public void A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_fil var section = document.Reader.Structure.Sections.Should().ContainSingle().Which; section.State.Should().Be(XRefSectionState.Malformed); section.Fault.Should().Be( - "its /Length holds the reference 4 0 R, which no section read before it places where it can be read, and no endstream follows its data before the end of the file"); + "its /Length is the reference 4 0 R, which no section read before it places where it can be read, and no endstream follows its data before the end of the file"); document.Diagnostics.Select(entry => entry.Code).Should().NotContain(code => code.StartsWith("limit.", StringComparison.Ordinal)); } @@ -2118,7 +2118,23 @@ public void A_deferred_length_s_data_with_no_endstream_before_the_next_object_ma document.Reader.Structure.Sections[1].Fault.Should().Be(string.Create( CultureInfo.InvariantCulture, - $"its /Length holds the reference 9 0 R, which no section read before it places where it can be read, and no endstream follows its data before the next object, at {next}")); + $"its /Length is the reference 9 0 R, which no section read before it places where it can be read, and no endstream follows its data before the next object, at {next}")); + } + + [Fact] + public void A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain() + { + // As above, the update placing object 9, which holds the reference 11 0 R, which no section places: the fault names + // the link the chain could not read, not the first, which it read (#220). + var file = ChainFiles.UpdateWithIndirectLength(spellingEndStream: false, olderIndirect: true, nine: "11 0 R"); + var at = (int)PdfTemplate.OffsetOf(file, "endstream"); + file[at] = (byte)'x'; + var next = PdfTemplate.OffsetOf(file, "8 0 obj\n(new)"); + using var document = PdfDocument.Open(file); + + document.Reader.Structure.Sections[1].Fault.Should().Be(string.Create( + CultureInfo.InvariantCulture, + $"its /Length holds the reference 11 0 R, which no section read before it places where it can be read, and no endstream follows its data before the next object, at {next}")); } [Theory] From 97b15cbebee16ac4f7bf963b61a3849bfaa3e766 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 11:05:50 +0000 Subject: [PATCH 037/146] fix: read where a stream's data starts past a guard's window that ends on its keyword A window that ended right after "stream", or on the carriage return after it, could not say where the data starts. A larger window was asked for, and at a guard's bound none followed: the parse was kept cut, its data starting a byte early on the end-of-line and running the declared length past the cut. No search for endstream was made and no endobj rule applied. A cross-reference stream so cut had its rows read misaligned, the index rebuilt and the catalog judged missing. The end-of-line after the keyword now counts toward no window, as the data does: the reader reads it from the file, and the stream reads as under a window one byte larger. A parser the reader does not back still asks for a larger buffer. The test that held the old shape as the guard's now has the window end inside the keyword. The guard cuts the object there, and nothing of the length is recorded, which is what it held. Refs #174 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 2 +- docs/website/docs/concepts/reader-limits.md | 3 +- .../Documents/PdfReaderLimits.cs | 6 +- .../IO/IPdfStreamDataProvider.cs | 8 +++ src/AdCodicem.Pdf/IO/PdfFileReader.cs | 18 ++++++ src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 16 +++-- .../AdCodicem.Pdf.Tests/StreamLengthTests.cs | 11 ++-- tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs | 63 +++++++++++++++++++ 8 files changed, 113 insertions(+), 14 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index 615d3f4e..dc2a0798 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -351,7 +351,7 @@ Known gaps: | A `/Length` naming an object a guard cut or left out of the index, judged by what the cut left of it: null, reported as no length, or the `1` it parted of `15`, reported as a length the data contradicts; or the stream's missing `endstream` left unreported along with the length | Keyed on the object the reference leads to, through a chain of references too, never on the value read (`StreamLengthForm.ObjectCut`, `ObjectUnindexed`): an integer read before the cut is only a place to look for `endstream` at, never kept as a length, and the data is otherwise found by its `endstream`. Nothing is said of the length, whose fault is the guard's, reported where it was reached; a missing `endstream` is still the file's, and reported, naming the object at the end of the chain of references, the one the guard cut or left out: "The stream's /Length names object N G, which one of the reader's limits cut, and no endstream follows its data …", or "…, which one of the reader's limits left out of the index, and no endstream follows its data …". A cross-reference stream's `/Length` so cut is left alike once the chain is read, and makes its section the guard's when no `endstream` ends its data, as *Indexing* says ([#224]) | `GuardCutValueTests.A_length_a_guard_cut_is_not_judged_and_the_data_is_found_by_its_endstream`, `A_length_a_guard_cut_leaves_a_missing_endstream_the_file_s_fault`, `A_length_an_object_stream_s_guard_left_unread_is_not_judged`, `A_length_whose_reference_leads_to_an_object_a_guard_left_out_of_the_index_is_not_judged`, `A_missing_endstream_names_the_object_a_guard_cut_at_the_end_of_the_length_s_chain_and_keeps_no_length`, `A_missing_endstream_is_the_file_s_fault_when_a_guard_left_the_length_out_of_the_index`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read` | No | | An indirect `/Length` leading back to its stream, or chained | The loading set and `MaxNestedLoads`, as above | `HostileInputTests.Survives_an_object_whose_length_refers_to_itself`, `Survives_a_chain_of_lengths_longer_than_the_stack_is_deep` | No | | A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | -| A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found: no window is grown to find an `endstream`, the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_data_start_a_limit_cuts_records_nothing_of_its_length`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows` | No | +| A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none` | Property | | A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more` | Plausibly | | Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, and by white space after a declared length that runs past a first read, about twice the file at most; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | diff --git a/docs/website/docs/concepts/reader-limits.md b/docs/website/docs/concepts/reader-limits.md index e4f7e145..43fb5af3 100644 --- a/docs/website/docs/concepts/reader-limits.md +++ b/docs/website/docs/concepts/reader-limits.md @@ -110,7 +110,8 @@ than a reference to a reference. An object stream's header lists its objects by one's objects each lie before the next one starts: the reader reads each no further, and a header that breaks that order ends where it breaks it. A sound stream's `endstream` follows its declared length; where it does not, or the `/Length` gives none, the reader searches the file for it past the bytes it parsed the object through, rather than parse -the object through a larger window, so that a stream's data never counts toward its object's length. A document's +the object through a larger window, so that a stream's data — and the end-of-line after its `stream` keyword — never +counts toward its object's length. A document's searches read four times the file at most: a valid file is read past those bytes only for the data of a cross-reference stream whose `/Length` the chain cannot read yet, and for white space and comments after a declared length, about twice the file. Those stay internal constants, since lifting them would let nothing more be read, and diff --git a/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs b/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs index 2b44454d..336dec9b 100644 --- a/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs +++ b/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs @@ -88,9 +88,9 @@ public sealed record PdfReaderLimits /// directly or an object stream holds it. Defaults to 16 MB, which a direct array of a million references would need. /// /// - /// A stream's data never counts toward it, however its end is found: a /Length that is wrong, missing or past - /// the file has the data searched for its endstream past the bytes the object was parsed through, not read - /// through a larger window (#174); what the reader reads of the data at once is + /// A stream's data never counts toward it, however its end is found, nor the end-of-line after its stream + /// keyword: a /Length that is wrong, missing or past the file has the data searched for its endstream past + /// the bytes the object was parsed through, not read through a larger window (#174); what the reader reads of the data at once is /// 's. A value above is taken as that length. Reaching it reports /// , once for each object, and for each object an object /// stream holds, once for each definition of that stream. diff --git a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs index 0091658c..6dc82974 100644 --- a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs +++ b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs @@ -19,6 +19,14 @@ internal interface IPdfStreamDataProvider /// Creates stream data for bytes at . PdfStreamData Create(long absoluteOffset, int length); + /// + /// Gives where the data of a stream starts in the file: past the end-of-line after its stream keyword, which ends + /// at . The parser asks when its buffer ends on the keyword, or on the carriage + /// return after it, and so cannot say: a window a guard ended there is not grown for it, since stream data counts + /// toward no window (#174). + /// + long DataStart(long absoluteKeywordEnd); + /// /// Determines whether an endstream keyword follows , after white space and /// comments, however many (#174), and what follows it. The parser asks at the end of a stream's declared length when its diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index fb8dcf0a..3412969a 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -1540,6 +1540,24 @@ PdfStreamData IPdfStreamDataProvider.Create(long absoluteOffset, int length) return new FileStreamData(this, absoluteOffset, Math.Clamp(length, 0, available), _guard); } + /// + long IPdfStreamDataProvider.DataStart(long absoluteKeywordEnd) + { + // The end-of-line after "stream" is a carriage return and a line feed, or either alone, as the lexer skips it. + Span endOfLine = stackalloc byte[2]; + var read = absoluteKeywordEnd < 0 || absoluteKeywordEnd >= _source.Length + ? 0 + : _source.Read(absoluteKeywordEnd, endOfLine[..(int)Math.Min(endOfLine.Length, _source.Length - absoluteKeywordEnd)]); + var skipped = read > 0 && endOfLine[0] == (byte)'\r' ? 1 : 0; + + if (skipped < read && endOfLine[skipped] == (byte)'\n') + { + skipped++; + } + + return absoluteKeywordEnd + skipped; + } + /// EndObjState? IPdfStreamDataProvider.CheckEndStream(long absoluteOffset) { diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index caa554bd..f821ac34 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -706,11 +706,19 @@ private PdfStream ReadStream(PdfDictionary dictionary) _endObjUnknown = false; // The end-of-line after "stream" is not data. A buffer that ends on the keyword, or on the carriage - // return of a CR LF, cannot say where the data starts; a larger one can — unless the buffer ends the data, and the - // stream's data, which starts there, holds nothing. + // return of a CR LF, cannot say where the data starts: the file says, as it says what follows the data, since the + // data counts toward no window — not even one a guard ended there (#174) —; a parser the reader does not back + // needs a larger buffer. One that ends the data says: the stream's data, which starts there, holds nothing. if (!_endsData && dataStart >= span.Length && (dataStart == keywordEnd || span[dataStart - 1] != (byte)'\n')) { - _truncated = true; + if (_streamData is null) + { + _truncated = true; + } + else + { + dataStart = (int)(_streamData.DataStart(_baseOffset + keywordEnd) - _baseOffset); + } } var declared = ReadLength(dictionary); @@ -743,7 +751,7 @@ private PdfStream ReadStream(PdfDictionary dictionary) var deferred = declared.Form == StreamLengthForm.Deferred; var declaredEnd = length < 0 ? -1 : _baseOffset + dataStart + length; var search = _streamData is not null - ? _streamData.FindEndStream(_objectNumber, _baseOffset + dataStart, declaredEnd, span[dataStart..], deferred) + ? _streamData.FindEndStream(_objectNumber, _baseOffset + dataStart, declaredEnd, span[Math.Min(dataStart, span.Length)..], deferred) : SearchBuffer(span, dataStart, declaredEnd, deferred); if (search is not { } settled) diff --git a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs index 731ff678..23a2542e 100644 --- a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs +++ b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs @@ -1178,17 +1178,18 @@ public void A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fa } [Fact] - public void A_stream_whose_data_start_a_limit_cuts_records_nothing_of_its_length() + public void A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length() { - // The window MaxObjectLength allows ends on the stream keyword: where the data starts is past it, the guard's, and - // what the parser found of the stream is the limit's. + // The window MaxObjectLength allows ends inside the stream keyword: whether a stream follows the dictionary is past + // it, the guard's, and what the parser found of the object is the limit's. A window that ends on the whole keyword + // reads the stream as a larger one does: its data counts toward no window (#174). var header = Invariant($"{StreamNumber} 0 obj\n"); var dictionary = "<< /Pad (" + new string('x', 1024 - header.Length - "<< /Pad () >>\nstream".Length) + ") >>\nstream"; var file = Document((StreamNumber, dictionary + "\n" + Data(100) + "\nendstream")); - var options = PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxObjectLength = 1024 } }; + var options = PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxObjectLength = 1024 - "eam".Length } }; using var document = PdfDocument.Open(file, options); - document.GetObject(new PdfObjectId(StreamNumber)).AsStream().Required(); + document.GetObject(new PdfObjectId(StreamNumber)); document.Diagnostics.Should().ContainSingle().Which.Code.Should().Be(PdfDiagnosticCodes.LimitObject); document.Reader.TryGetStreamLengthFault(StreamNumber, out _).Should().BeFalse(); diff --git a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs index 7bcc3491..cc637cfe 100644 --- a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs +++ b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs @@ -4,6 +4,7 @@ using AdCodicem.Pdf.Documents; using AdCodicem.Pdf.IO; using AdCodicem.Pdf.Objects; +using AdCodicem.Pdf.Validation; namespace AdCodicem.Pdf.Tests; @@ -142,6 +143,68 @@ public void A_stream_ending_just_before_the_window_edge_is_confirmed_without_a_l failures.Should().BeEmpty($"{Describe(endOfLine)} must not cost a larger window"); } + [Theory] + [InlineData("\n", true)] + [InlineData("\r\n", true)] + [InlineData("\r", true)] + [InlineData("\n", false)] + [InlineData("\r\n", false)] + public void A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword(string endOfLine, bool ended) + { + // MaxObjectLength ends the window right after "stream", or after its carriage return: the dictionary is whole, and + // the end-of-line, as the data, counts toward no window. The data starts past it, as under a window one byte + // larger — never on it, a byte early, nor at the declared length past the cut with no search (#174). + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + [4] = file.Object( + 4, + $"<< /Pad ({new string('x', 60)}) /Length {(ended ? 40 : 5000)} >>\nstream{endOfLine}" + new string('a', 40) + + (ended ? "\nendstream" : string.Empty)), + [6] = file.Object(6, "(six)"), + }; + file.Table(7, rows, "/Size 7 /Root 1 0 R"); + var bytes = file.Bytes; + var header = PdfTemplate.OffsetOf(bytes, "4 0 obj"); + var keywordEnd = PdfTemplate.OffsetOf(bytes, "stream" + endOfLine) + "stream".Length; + var window = (int)(keywordEnd - header) + (endOfLine == "\r\n" ? 1 : 0); + + using var whole = PdfDocument.Open(bytes); + using var guarded = PdfDocument.Open( + bytes, + PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxObjectLength = window } }); + + var expected = whole.GetObject(new PdfObjectId(4)).AsStream().Required().GetRawBytes().ToArray(); + guarded.GetObject(new PdfObjectId(4)).AsStream().Required().GetRawBytes().ToArray().Should().Equal(expected); + expected.Should().Equal(Encoding.ASCII.GetBytes(new string('a', 40))); + guarded.Diagnostics.Select(entry => entry.Message).Should().Equal(whole.Diagnostics.Select(entry => entry.Message)); + } + + [Fact] + public void A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword() + { + // MaxTrailerLength ends the window right after the stream's keyword: its dictionary is whole, and its rows are read + // past it, as under a window one byte larger — not a byte early, misaligned, which rebuilt the index and judged the + // catalog missing (#174). + var bytes = new TestPdfBuilder() + .WithObject(1, "<< /Type /Catalog /Pages 2 0 R >>") + .WithObject(2, "<< /Type /Pages /Kids [] /Count 0 >>") + .BuildWithXRefStream(rootNumber: 1); + var dictionary = PdfTemplate.OffsetOf(bytes, "/Type /XRef"); + var header = Encoding.Latin1.GetString(bytes).LastIndexOf(" 0 obj", (int)dictionary, StringComparison.Ordinal) - 1; + var keywordEnd = Encoding.Latin1.GetString(bytes).IndexOf("stream", (int)dictionary, StringComparison.Ordinal) + "stream".Length; + + using var document = PdfDocument.Open( + bytes, + PdfReaderOptions.Default with { Limits = PdfReaderLimits.Default with { MaxTrailerLength = keywordEnd - header } }); + + document.Diagnostics.Should().BeEmpty(); + document.WasRepaired.Should().BeFalse(); + new PdfValidator().Validate(document).Findings.Should().BeEmpty(); + } + [Fact] public void An_anomaly_inside_an_object_longer_than_the_window_is_reported_once() { From ce3deccd889c7519df46eadbe6dcf159ff33bc6b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 11:09:53 +0000 Subject: [PATCH 038/146] test: hold that a chain a guard ends leaves the rules that need it whole unjudged The test of a /Prev or /XRefStm a guard cut checked the validation's report on the hidden sections, but not that file.size-wrong and page-tree.page-orphaned leave such a chain alone: on its file neither would have spoken either way. A file whose older table alone indexes the highest number, and whose update writes a page the tree does not list, has its /Size accepted and the page said to be left out, read whole, and neither judged, the older table hidden. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 29 +++++++++++++++++++ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 23 +++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 0becf2be..bbb662f9 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -432,6 +432,35 @@ public static byte[] UpdateNamingItsPrevious(string prev, int padding = 0) return file.ToArray(); } + /// + /// A file of two classic tables. The first indexes the catalog, the page tree, the page and object 12, the highest + /// number; the second, an update, writes the catalog, the page tree and the page again, indexes them with object 4, a + /// page the tree does not list, and object 9, which holds the first table's offset after + /// spaces, and names that table through /Prev 9 0 R. Both trailers give /Size 13. + /// + public static byte[] UpdateHidingItsHighestNumber(int padding) + { + using var file = new Writer("1.4"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + file.Object(12, "(twelve)"); + var first = file.Position; + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"xref\n0 4\n0000000000 65535 f\r\n{file.OffsetOf(1):D10} 00000 n\r\n{file.OffsetOf(2):D10} 00000 n\r\n{file.OffsetOf(3):D10} 00000 n\r\n12 1\n{file.OffsetOf(12):D10} 00000 n\r\ntrailer\n<< /Size 13 /Root 1 0 R >>\nstartxref\n{first}\n%%EOF\n")); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + file.Object(4, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + file.Object(9, new string(' ', padding) + first.ToString(CultureInfo.InvariantCulture)); + var second = file.Position; + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"xref\n1 4\n{file.OffsetOf(1):D10} 00000 n\r\n{file.OffsetOf(2):D10} 00000 n\r\n{file.OffsetOf(3):D10} 00000 n\r\n{file.OffsetOf(4):D10} 00000 n\r\n9 1\n{file.OffsetOf(9):D10} 00000 n\r\ntrailer\n<< /Size 13 /Root 1 0 R /Prev 9 0 R >>\nstartxref\n{second}\n%%EOF\n")); + return file.ToArray(); + } + private enum Rows { Raw, diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index f58fceac..97e5ed65 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -550,6 +550,29 @@ public void A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s(string ke throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); } + [Fact] + public void A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain() + { + // Read whole, the update's /Size 13 is one more than object 12, which only the older table indexes, and object 4 is + // a page the tree does not list. With the older table hidden by the guard, object 12 is unknown: /Size is not judged + // against the 9 the update alone gives, nor is a page judged left out of the tree (#263). + var file = ChainFiles.UpdateHidingItsHighestNumber(padding: 3000); + + using (var whole = PdfDocument.Open(file)) + { + var report = new PdfValidator().Validate(whole); + report.Contains(PdfValidationRuleIds.FileSizeWrong).Should().BeFalse(); + report.Contains(PdfValidationRuleIds.PageTreePageOrphaned).Should().BeTrue(); + } + + using var guarded = PdfDocument.Open(file, ObjectGuard); + var findings = new PdfValidator().Validate(guarded).Findings.Select(finding => finding.RuleId).ToArray(); + + findings.Should().Contain(PdfValidationRuleIds.XRefCheckedInPart); + findings.Should().NotContain(PdfValidationRuleIds.FileSizeWrong); + findings.Should().NotContain(PdfValidationRuleIds.PageTreePageOrphaned); + } + [Theory] [InlineData(false)] [InlineData(true)] From 4a761b1e754927f3977b3e20a5cf654b20773632 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 11:11:38 +0000 Subject: [PATCH 039/146] docs: say what a stream's raw and decoded bytes are and where their guard is reported The texts of limit.stream-length and MaxStreamLength still said it is reported once for each stream; it is, in the document's diagnostics, and in diagnostics of one's own each time the stream is decoded. Decode's result is a view of the memory a document was opened from wherever no filter changes the data, as the raw bytes are: Decode, PdfDocument and the object-model reference say so now. The raw bytes' summary no longer says "exactly as they appear in the file", the bound cutting them, and their remarks hold for a stream read from a document. Decode's remarks lose a missing verb, and the ThrowOnLimit table names loading a value the filters read under Decode, as under HasImageFilter. Refs #170 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/website/docs/reference/object-model.md | 4 +++- docs/website/docs/reference/reader-limits.md | 2 +- src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs | 5 +++-- src/AdCodicem.Pdf/Documents/PdfDocument.cs | 4 +++- src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs | 4 +++- src/AdCodicem.Pdf/Objects/PdfStream.cs | 10 ++++++---- src/AdCodicem.Pdf/Objects/PdfStreamData.cs | 5 +++-- .../Objects/PdfStreamDecodingExtensions.cs | 12 ++++++++---- 8 files changed, 30 insertions(+), 16 deletions(-) diff --git a/docs/website/docs/reference/object-model.md b/docs/website/docs/reference/object-model.md index 7d5c562b..50494c13 100644 --- a/docs/website/docs/reference/object-model.md +++ b/docs/website/docs/reference/object-model.md @@ -116,4 +116,6 @@ An object keeps what it holds once its document is disposed: a dictionary's dire the document was opened and whether or not it was read before: `Resolve()` on a reference, and every accessor that resolves one, such as `GetDictionary`; `GetRawBytes()` and `Decode()` on a stream read from the file, and `HasImageFilter()` when it has a value to load. Raw bytes taken before from a document held in memory stay a view of -that memory: they keep it alive for as long as they are held. +that memory, and so does what `Decode()` gave back when no filter changed the data — with no filter, or where decoding +stopped at the first: an image filter, one the library does not know or could not decode the data with, the end of the +chain —: they keep it alive for as long as they are held, and change as it does. diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index 6326dbce..848a4e6f 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -157,7 +157,7 @@ The exception comes from whichever operation reaches the limit: | `PdfDocument.Open` | While it indexes the file and reads its catalog | | `GetObject`, or resolving a reference | When an object is too long, or the object stream that holds it runs past `MaxStreamLength`, decodes too far or names too many filters; when the read rebuilds the index, at any limit the rebuild reaches, once the rebuild is finished — the first reached: a limit the rebuild's scan reaches before the read's own, the read's own before one finishing the rebuild reaches; and each time it looks up an object a full index left out | | `GetRawBytes` | When the stream's data runs past `MaxStreamLength`, each time it is read | -| `Decode` | When a stream's data runs past `MaxStreamLength`, the stream decodes too far, or its decoding reaches the filter past `MaxFilterChainLength` | +| `Decode` | When a stream's data runs past `MaxStreamLength`, the stream decodes too far, its decoding reaches the filter past `MaxFilterChainLength`, or loading a value the filters read — the `/Filter` entry, a filter's parameters — reaches a limit, as resolving a reference does | | `HasImageFilter` | When loading a value decoding reads before the image filter — the `/Filter` entry, a filter's parameters — reaches a limit, as resolving a reference does | | `PdfValidator.Validate` | As it reads, like any other caller; and before its first rule, at any limit the rebuild it may run then reaches, once that rebuild is finished | diff --git a/src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs b/src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs index da088a0e..fa237771 100644 --- a/src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs +++ b/src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs @@ -225,8 +225,9 @@ public static class PdfDiagnosticCodes /// /// A stream's encoded data runs past , and was read up to it: /// what lies within it is decoded as data the reader cut, and handed back as it is where decoding hands its data back - /// undecoded. Reported once for each stream, when its data is first read. Like every limit.* code, the guard is - /// the reader's, not damage in the file. + /// undecoded. Reported once for each stream in the document's diagnostics, when its data is first read, and in + /// diagnostics of your own each time you decode it. Like every limit.* code, the guard is the reader's, not + /// damage in the file. /// public const string LimitStreamLength = "limit.stream-length"; diff --git a/src/AdCodicem.Pdf/Documents/PdfDocument.cs b/src/AdCodicem.Pdf/Documents/PdfDocument.cs index bd38164f..0e7e95b6 100644 --- a/src/AdCodicem.Pdf/Documents/PdfDocument.cs +++ b/src/AdCodicem.Pdf/Documents/PdfDocument.cs @@ -29,7 +29,9 @@ namespace AdCodicem.Pdf.Documents; /// every object's ToString. Disposing lets go of what the document kept: the objects it read, its index, and the /// file, closed when it was opened from a path, and held no longer when it was opened from memory or a stream. A stream's /// raw bytes taken before from a document held in memory are a view of that memory rather than a copy -/// (): for as long as they are held, they keep it alive, and they change as it does. +/// (), and so is what decoding gives back when no filter changes the data +/// (): for as long as they are held, they keep it alive, and they change +/// as it does. /// /// public sealed class PdfDocument : IDisposable diff --git a/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs b/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs index 336dec9b..e6293fe9 100644 --- a/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs +++ b/src/AdCodicem.Pdf/Documents/PdfReaderLimits.cs @@ -72,7 +72,9 @@ public sealed record PdfReaderLimits /// decoding it —, so a value above is taken as that length, and a /Length past it /// is cut there. Data past the bound is cut at it: what lies within it is read, and decoded as data the reader cut, /// whose lost tail is no damage. Reaching it reports , - /// once for each stream. stays the length the file holds. + /// once for each stream in the document's diagnostics, and in diagnostics of your own each time you decode the stream + /// (). stays the + /// length the file holds. /// /// /// Data that decoding hands back as it is — no filter, an image filter, a filter the library does not know, one that diff --git a/src/AdCodicem.Pdf/Objects/PdfStream.cs b/src/AdCodicem.Pdf/Objects/PdfStream.cs index 8d832e13..2a1e8b07 100644 --- a/src/AdCodicem.Pdf/Objects/PdfStream.cs +++ b/src/AdCodicem.Pdf/Objects/PdfStream.cs @@ -29,12 +29,14 @@ public PdfStream(PdfDictionary dictionary, PdfStreamData data) /// Gets the number of encoded bytes. public int RawLength => Data.Length; - /// Returns the encoded bytes, exactly as they appear in the file. + /// Returns the encoded bytes, as they appear in the file, up to the reader's bound. /// - /// The bytes are read from the document's source each time they are asked for, and kept by nothing: a document held in - /// memory lends them where they lie, and any other source is read again. No more than + /// A stream read from a document has its bytes read from the document's source each time they are asked for, and kept + /// by nothing: a document held in memory lends them where they lie — a view of that memory, which keeps it alive and + /// changes as it does —, and any other source is read again. No more than /// bytes are read: past it, what lies within it comes back, and - /// is reported once for the stream. + /// is reported once for the stream. A stream built over + /// bytes in memory gives them back as they are. /// /// The stream's document is disposed. /// diff --git a/src/AdCodicem.Pdf/Objects/PdfStreamData.cs b/src/AdCodicem.Pdf/Objects/PdfStreamData.cs index b4487224..5c0db855 100644 --- a/src/AdCodicem.Pdf/Objects/PdfStreamData.cs +++ b/src/AdCodicem.Pdf/Objects/PdfStreamData.cs @@ -31,9 +31,10 @@ public abstract class PdfStreamData /// internal virtual bool CutByGuard => false; - /// Returns the encoded bytes, exactly as they appear in the file. + /// Returns the encoded bytes, as they appear in the file, up to the reader's bound. /// - /// Data read from a document gives no more than bytes: past it, + /// Data read from a document held in memory is a view of that memory, which keeps it alive and changes as it does. Data + /// read from a document gives no more than bytes: past it, /// what lies within it comes back, fewer bytes than says, and /// is reported once in the document's diagnostics. /// diff --git a/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs b/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs index 6149b80f..8b93ec05 100644 --- a/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs +++ b/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs @@ -19,7 +19,10 @@ public static class PdfStreamDecodingExtensions /// /// /// Decoding stops at an image filter: a stream encoded with DCTDecode comes back as the JPEG it - /// already is, rather than as pixels nobody asked for. + /// already is, rather than as pixels nobody asked for. Data no filter changes comes back as + /// gives it — with no filter, or where decoding stops at the first: an image filter, + /// one the library does not know or could not decode the data with, the end of the chain —: from a document held in + /// memory, a view of that memory, which keeps it alive and changes as it does. /// /// /// Data that is damaged decodes as far as it can, and what decoded is kept: a Flate stream that lost its @@ -42,9 +45,10 @@ public static class PdfStreamDecodingExtensions /// reported once as ; the filters before the last share the /// bound for what they pass along together (). The /// filters after one the bound stopped decode what it kept, as data the reader cut: that it runs out is not - /// reported as damage. So do they data that runs past , of which - /// the reader reads what lies within it, reported as : once in the - /// document's diagnostics, and in diagnostics of your own each time the stream is decoded. A /Filter array that + /// reported as damage. So do they decode data that runs past + /// , of which the reader reads what lies within it, reported as + /// : once in the document's diagnostics, and in diagnostics of your + /// own each time the stream is decoded. A /Filter array that /// names more filters than /// ends decoding before the first past them, reported as /// when decoding reaches it. From 4de73fcc460a7a977a34129781cb3446e09bd876 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 11:16:39 +0000 Subject: [PATCH 040/146] fix: take no endobj a name or a comment holds for the one that ends a stream's data A stream with no endstream ended its data at its object's endobj when nothing but white space and comments followed it before the search's stop. The keyword was taken whole whenever the byte before it was no regular character. "/endobj" is a name, and "%endobj" a comment: each ended the data at the delimiter, and the object was said to have its endobj. A solidus or a percent sign before the keyword now makes it none, as a regular character does. The data then runs to the stop, as with no endobj. Other delimiters still end a token or stand alone; the bytes before the keyword are the data's, which no lexing reads. The summary of the bytes kept from one read to the next now says what they hold. Refs #174 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/architecture.md | 3 ++- src/AdCodicem.Pdf/IO/EndObjTail.cs | 13 +++++++++++-- tests/AdCodicem.Pdf.Tests/ParserTests.cs | 7 ++++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 7e32c5bc..83337fc6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -112,7 +112,8 @@ separate types, separate severity scales, and vocabularies that never share a co crafted file can still make it depend on the number a stream is first read under, on the rebuild's scan for trailers, or, once the searches of a document have read four times the file, on which streams were searched first ([#138], M23). With no `endstream` before the stop, the object's `endobj` ends the data where it - follows the declared length as an `endstream` would, or where it is the last one, a whole keyword, and only + follows the declared length as an `endstream` would, or where it is the last one, a whole keyword — no regular + character, solidus or percent sign before it, which would make it part of a token, a name or a comment —, and only white space and comments run from it to the stop. That last `endobj` is followed forward through the search's own reads, which read every byte up to the stop already: no byte before the stop is read a second time, backward, and a comment, which can be told only from the start of its line, is read as one. Without either, diff --git a/src/AdCodicem.Pdf/IO/EndObjTail.cs b/src/AdCodicem.Pdf/IO/EndObjTail.cs index 6b7ad748..589c0525 100644 --- a/src/AdCodicem.Pdf/IO/EndObjTail.cs +++ b/src/AdCodicem.Pdf/IO/EndObjTail.cs @@ -21,7 +21,10 @@ namespace AdCodicem.Pdf.IO; /// internal struct EndObjTail { - /// The most bytes before a keyword that what it ends needs: the byte before it, and an end-of-line of two. + /// + /// The bytes a read keeps of the reads before it: the part of an endobj they hold, five bytes at most, and the two + /// before it that say whether it starts a token and where its end-of-line starts — seven, rounded up. + /// private const int Kept = 8; private readonly long _dataStart; @@ -187,10 +190,16 @@ private int Straddling(ReadOnlySpan bytes, long at, int start, int end) /// Takes the endobj that starts at , unless the byte before it makes it part of a longer /// token, as the one the data may end at; whether the byte after it does is read next. /// + /// + /// A regular character before it makes it part of the token that character is in; a solidus makes it a name, + /// /endobj, and a percent sign part of a comment. The bytes before the keyword are the data's, which no lexing + /// reads: any other delimiter ends a token or stands alone, wherever a string or a dictionary it may open would end. + /// /// Whether the keyword starts a token. private bool Found(ReadOnlySpan bytes, long at, int start, long keyword) { - if (ByteAt(bytes, at, start, keyword - 1) is var before and >= 0 && PdfCharacters.IsRegular((byte)before)) + if (ByteAt(bytes, at, start, keyword - 1) is var before and >= 0 && + (PdfCharacters.IsRegular((byte)before) || before is '/' or '%')) { return false; } diff --git a/tests/AdCodicem.Pdf.Tests/ParserTests.cs b/tests/AdCodicem.Pdf.Tests/ParserTests.cs index f8b176a1..2ce37743 100644 --- a/tests/AdCodicem.Pdf.Tests/ParserTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ParserTests.cs @@ -186,10 +186,15 @@ public void Ends_the_containers_an_object_left_open_at_the_next_object_s_header_ [InlineData("hello endobjx\n6 0 obj\n", -1)] [InlineData("helloendobj\n6 0 obj\n", -1)] [InlineData("endobj\n6 0 obj\n", 0)] + [InlineData("hello\n/endobj\n6 0 obj\n", -1)] + [InlineData("hello\n%endobj\n6 0 obj\n", -1)] + [InlineData("hello\n)endobj\n6 0 obj\n", 7)] + [InlineData("hello\n[endobj\n6 0 obj\n", 7)] public void Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow(string data, int length) { // Parsed alone, a stream with no endstream ends where its object's endobj ends it, a comment after which is white - // space; a token after it leaves the data running to the next object's header (#174). + // space; a token after it leaves the data running to the next object's header (#174). A name or a comment that the + // keyword would be part of makes it none: "/endobj" is a name. var parser = new PdfObjectParser(Encoding.ASCII.GetBytes("4 0 obj\n<< /Length 200 >>\nstream\n" + data)); parser.TryReadIndirectObject(out _, out var value).Should().BeTrue(); From c4bf5b306cf2ed4b09cdf732e293b66163dce945 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 11:21:35 +0000 Subject: [PATCH 041/146] fix: leave rows past a spelled endstream to the guard wherever the cut length lies A cross-reference stream whose /Length the chain could not read, and whose rows spell endstream, has its data taken again only where the section's own row places the length. Where a guard cut that length, the rows past the spelled endstream were the guard's only when that row placed the cut object. A newer section placing it, or the object the row places holding a reference to a cut object, charged them to the file instead. The section was incomplete, the index rebuilt, xref.section-malformed reported. Both now leave the rows to the guard: the section counts as cut, the chain ends there, nothing is rebuilt, and the guard is reported once the chain is read, or thrown by Open. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 16 +++++-- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 42 +++++++++++++++++++ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 30 +++++++++++++ 3 files changed, 84 insertions(+), 4 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 3412969a..cece65f1 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3827,9 +3827,11 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect /// reserved type is ignored there, as the specification says (). The object is read /// where that row places it as reads one: nothing is corrected, cached, recorded or /// reported. One a guard cuts gives what the cut left of it, an integer read before the cut only a place to look for - /// endstream at (#224), and says so when no endstream confirms it. A row that - /// does not stand — a newer section gave the number, or refused it; the row lies past those held, places the object in - /// an object stream, or under another generation — leaves the data as it was taken. + /// endstream at (#224), and says so when no endstream confirms it — as it does + /// when the object holds a reference that leads to an object a guard cut, or when a newer section gave the number and + /// the object it places is one, or leads to one. A row that does not stand otherwise — a newer section gave the number, + /// or refused it; the row lies past those held, places the object in an object stream, or under another generation — + /// leaves the data as it was taken. /// private PdfStream? RetakeDeferred( PdfStream stream, @@ -3846,6 +3848,9 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect if (_xref.TryGet(number, out _) || _refusedNumbers?.Contains(number) == true) { + // The object a newer section places, which the chain could not read: a guard that cut it, or the object a chain + // of references from it leads to, leaves how far the data runs unknown. + lengthCut = LeadsToCut(stream.Dictionary.GetRaw(PdfName.Length)); return null; } @@ -3894,8 +3899,11 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect return new PdfStream(stream.Dictionary, ((IPdfStreamDataProvider)this).Create(dataStart, (int)length)); } - lengthCut = pastGuard; + lengthCut = pastGuard || (value is PdfReference && LeadsToCut(value)); return null; + + // Whether a value read through the sections read so far leads to an object a guard cut. + bool LeadsToCut(PdfObject? written) => Follow(PdfName.Length, written, entry: true, out _, out var cut) is not null && cut is not null; } /// diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index bbb662f9..8ae09bb8 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -432,6 +432,48 @@ public static byte[] UpdateNamingItsPrevious(string prev, int padding = 0) return file.ToArray(); } + /// + /// A file of two revisions. The first holds the catalog, the page tree and the page, and cross-reference stream 7, whose + /// /Length names object 5, which holds the data's length after 3,000 spaces; or, when , + /// object 4, which holds 9 0 R, object 9 holding the length so. Its rows of objects 8 and 9 — 10 and 11 when + /// —, of reserved types, spell endstream after an end-of-line, and the page's row + /// comes after them. The second is a classic table that indexes object 5 — 9 when — again, + /// where it is, and names stream 7 through /Prev. The file is valid. + /// + public static byte[] UpdatePlacingAPaddedLength(bool chained) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + var lengthNumber = chained ? 9 : 5; + var spelled = "\nendstream\0\0\0\0"u8.ToArray(); + file.Object(lengthNumber, new string(' ', 3000) + (6 * 7 + spelled.Length).ToString(CultureInfo.InvariantCulture)); + + if (chained) + { + file.Object(4, "9 0 R"); + } + + var stream = file.Position; + var data = new List(Encode( + [(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(chained ? 4 : 5), 0), (1, stream, 0)], + Rows.Raw)); + data.AddRange(spelled); + data.AddRange(Encode([(1, file.OffsetOf(3), 0)], Rows.Raw)); + var index = chained ? "[0 3 4 1 7 1 10 2 3 1]" : "[0 3 5 1 7 1 8 2 3 1]"; + file.Stream( + 7, + string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size {(chained ? 12 : 10)} /W [1 4 2] /Index {index} /Root 1 0 R /Length {(chained ? 4 : 5)} 0 R"), + [.. data]); + file.Text("%" + new string('-', 1200) + "\n"); + var xref = file.Position; + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"xref\n0 1\n0000000000 65535 f\r\n{lengthNumber} 1\n{file.OffsetOf(lengthNumber):D10} 00000 n\r\ntrailer\n<< /Size {(chained ? 12 : 10)} /Root 1 0 R /Prev {stream} >>\nstartxref\n{xref}\n%%EOF\n")); + return file.ToArray(); + } + /// /// A file of two classic tables. The first indexes the catalog, the page tree, the page and object 12, the highest /// number; the second, an update, writes the catalog, the page tree and the page again, indexes them with object 4, a diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 97e5ed65..2480f907 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -521,6 +521,36 @@ public void Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_de new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().BeFalse(); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut(bool chained) + { + // Stream 7's /Length leads to an object the newer table places — object 5 itself, or object 9 through object 4, + // which only stream 7's own row places — and MaxObjectLength cuts before its value: the rows past the endstream they + // spell are the guard's, as when the cut object is the one the section's own row places (#224). Read whole, the + // file is clean. + var file = ChainFiles.UpdatePlacingAPaddedLength(chained); + + using (var whole = PdfDocument.Open(file)) + { + whole.GetObject(new PdfObjectId(3)).AsDictionary().Should().NotBeNull(); + whole.Diagnostics.Should().BeEmpty(); + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + var section = document.Reader.Structure.Sections[1]; + section.CutByLimit.Should().BeTrue(); + section.Incomplete.Should().BeFalse(); + new PdfValidator().Validate(document).Findings.Should().NotContain(finding => finding.Severity == PdfValidationSeverity.Error); + + var throwing = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); + throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); + } + [Theory] [InlineData("Prev")] [InlineData("XRefStm")] From 5d31559555dc6c96bfabdcb36555cf8e5c3d0e1e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 11:33:38 +0000 Subject: [PATCH 042/146] fix: follow a chain to the object a guard cut as far as its value is read A value is read through a chain of references as far as the read resolves it: 31 objects for a reference, 32 for a stream's /Length and a trailer's /Prev or /XRefStm, read from the object a reference names and then resolved. The checks of a guard's cut stopped short of that: - the target a /Length's chain led to was taken at the 31st object, unloaded, so a cut there was charged to the file, and a cut 32nd never seen; - the chain read followed a cross-reference stream's /Length through 31 objects, and the parse then went on to read a cut 32nd as null, judged the file's length; - a /Prev or /XRefStm cut at its 32nd object was no offset, and the index was rebuilt. Each now follows as far as the value is read, the chain resolved before the guard is asked of its target, and a chain longer than that reads null where the read gives null. A deferred /Length that the chain resolves is described by what it resolves to, as the parse describes it, not by the reference its first object holds. The texts of the forms a guard leaves a /Length in say the object named is the chain's end. Refs #224, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 70 +++++++--- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 4 +- src/AdCodicem.Pdf/IO/StreamLengthFault.cs | 5 +- src/AdCodicem.Pdf/IO/StreamLengthForm.cs | 6 +- src/AdCodicem.Pdf/Objects/PdfReference.cs | 9 +- .../DocumentReaderTests.cs | 14 ++ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 122 ++++++++++++++++++ 7 files changed, 203 insertions(+), 27 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index cece65f1..c2c85834 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -195,6 +195,14 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// private const int MaxReferenceLinks = 31; + /// + /// How many references are followed from a value read from the object a reference names and then resolved, as a stream's + /// /Length is parsed and a trailer's /Prev and /XRefStm are read: one more than + /// , the first object's value resolving through as many as a reference does. No valid file + /// reaches it either. + /// + private const int LengthLinks = MaxReferenceLinks + 1; + private static ReadOnlySpan ObjKeyword => "obj"u8; private static ReadOnlySpan TrailerKeyword => "trailer"u8; private static ReadOnlySpan StartXRefKeyword => "startxref"u8; @@ -752,16 +760,17 @@ public PdfObject GetObject(PdfObjectId id) } /// - /// Reads, while the chain is read, what object leads to through as many references as - /// follows, each read with ; null when one of them cannot be read - /// — no section read so far places it, or a guard cut it —: a value whose chain leads there waits for the chain, as one - /// the first reference cannot read does. + /// Reads, while the chain is read, what the /Length object leads to through as many + /// references as the parse resolves a /Length through (), each read with + /// ; null when one of them cannot be read — no section read so far places it, or a guard + /// cut it —: a value whose chain leads there waits for the chain, as one the first reference cannot read does. A value + /// still a reference past them is one the parse reads as null. /// private PdfObject? ReadThroughWithoutLoading(PdfObjectId id) { var read = ReadWithoutLoading(id); - for (var link = 1; read is PdfReference next && link < MaxReferenceLinks; link++) + for (var link = 1; read is PdfReference next && link < LengthLinks; link++) { read = ReadWithoutLoading(next.Id); } @@ -1410,9 +1419,11 @@ private static bool Consume(HeaderReading reading, ref long consumed, long end) /// past the cut, or is unknown, and is the guard's until the guard is lifted, rather than a value the file gave /// (#224, #263). Call it once the reference is resolved. /// - internal bool LeftByGuard(PdfReference reference) + /// The reference, as the file wrote it. + /// How many objects its chain is followed through (). + internal bool LeftByGuard(PdfReference reference, int objects = PdfReference.MaxChainLength - 1) { - var target = reference.Target().Number; + var target = reference.Target(objects).Number; return IsCutAtLimit(target) || GetPresence(target) == ObjectPresence.Unindexed; } @@ -1842,7 +1853,9 @@ PdfObject IPdfStreamDataProvider.ResolveLength(PdfReference reference, out Objec // read once the chain is read: the data is found by its endstream meanwhile, and the length checked then (#182). var read = ReadThroughWithoutLoading(id); presence = read is null ? ObjectPresence.Deferred : ObjectPresence.Defined; - return read ?? PdfNull.Instance; + + // A chain longer than the parse resolves a /Length through gives null, as it does once the chain is read. + return read is null or PdfReference ? PdfNull.Instance : read; } var reentries = _reentries; @@ -1855,8 +1868,14 @@ PdfObject IPdfStreamDataProvider.ResolveLength(PdfReference reference, out Objec // left it out of the index: a value the cut left, null or an integer it parted, is none the file gave (#224). if (presence != ObjectPresence.Unindexed) { - // The object is not loaded again unless its value leads further. - var target = value is PdfReference ? reference.Target().Number : id.Number; + // The object is not loaded again unless its value leads further: the chain is resolved first, as the parse + // resolves it, so that the object it leads to is read when it is asked whether a guard cut it. + if (value is PdfReference next) + { + next.Resolve(); + } + + var target = value is PdfReference ? reference.Target(PdfReference.MaxChainLength).Number : id.Number; presence = IsCutAtLimit(target) ? ObjectPresence.Cut : target != id.Number && GetPresence(target) == ObjectPresence.Unindexed ? ObjectPresence.Unindexed : presence; @@ -2169,22 +2188,26 @@ private void CheckDeferredLengths() var (number, generation) = (deferred.Length.Number, deferred.Length.Generation); string? message = null; + // The chain is resolved as the parse resolves a /Length, one object further than a reference: the object it + // leads to is read before it is asked whether a guard cut it, and what it holds is what is judged. + var resolved = value.Resolve(); + // A /Length a guard cut or left out of the index — or the object a chain of references from it leads to — is // unknown, not wrong: the guard's report says why, and what the cut left of it, null or a parted integer, is // none the file gave (#224). - if (LeftByGuard(new PdfReference(deferred.Length, this))) + if (LeftByGuard(new PdfReference(deferred.Length, this), PdfReference.MaxChainLength)) { continue; } - if (value.AsInteger() is not { } length || length is < 0 or > int.MaxValue) + if (resolved.AsInteger() is not { } length || length is < 0 or > int.MaxValue) { var presence = value is PdfNull && reentries != _reentries ? ObjectPresence.Unproduced : GetPresence(number); var held = presence switch { ObjectPresence.Missing => "which the file lacks", ObjectPresence.Unproduced => "which could not be read", - _ => "which holds " + DescribeValue(value) + ", not a length", + _ => "which holds " + DescribeValue(resolved) + ", not a length", }; message = string.Create( CultureInfo.InvariantCulture, @@ -3587,7 +3610,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect { // The fault names the link of the chain that could not be read — the one no section read so far places, or the // one a guard cut —, which the chain read without loading stopped at, as Follow does. - var lengthFault = Follow(PdfName.Length, dictionary.GetRaw(PdfName.Length), entry: true, out _, out var cutObject); + var lengthFault = Follow(PdfName.Length, dictionary.GetRaw(PdfName.Length), entry: true, out _, out var cutObject, LengthLinks); if (cutObject is { } past) { @@ -3850,7 +3873,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect { // The object a newer section places, which the chain could not read: a guard that cut it, or the object a chain // of references from it leads to, leaves how far the data runs unknown. - lengthCut = LeadsToCut(stream.Dictionary.GetRaw(PdfName.Length)); + lengthCut = LeadsToCut(stream.Dictionary.GetRaw(PdfName.Length), LengthLinks); return null; } @@ -3899,11 +3922,13 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect return new PdfStream(stream.Dictionary, ((IPdfStreamDataProvider)this).Create(dataStart, (int)length)); } - lengthCut = pastGuard || (value is PdfReference && LeadsToCut(value)); + // The object read holds the chain's second link: the chain goes on as far as the parse resolves a /Length through. + lengthCut = pastGuard || (value is PdfReference && LeadsToCut(value, LengthLinks - 1)); return null; // Whether a value read through the sections read so far leads to an object a guard cut. - bool LeadsToCut(PdfObject? written) => Follow(PdfName.Length, written, entry: true, out _, out var cut) is not null && cut is not null; + bool LeadsToCut(PdfObject? written, int links) => + Follow(PdfName.Length, written, entry: true, out _, out var cut, links) is not null && cut is not null; } /// @@ -4010,14 +4035,19 @@ public bool TryRead(PdfName key, PdfObject? written, bool entry, out PdfObject? /// Whether is the entry's own value, rather than an element of it. /// The value it leads to, or null when it leads to none. /// The object a guard cut that left the value unread, or null when none did. - private string? Follow(PdfName key, PdfObject? written, bool entry, out PdfObject? value, out PdfObjectId? cut) + /// + /// How many references are followed: one more for a value read from the object a reference names and then resolved — + /// a stream's /Length, a trailer's /Prev or /XRefStm —, as says. + /// + private string? Follow( + PdfName key, PdfObject? written, bool entry, out PdfObject? value, out PdfObjectId? cut, int links = MaxReferenceLinks) { value = written; cut = null; for (var link = 0; value is PdfReference reference; link++) { - if (link == MaxReferenceLinks) + if (link == links) { value = null; return $"its {FileQuote.Name(key)} leads from reference to reference without reaching a value"; @@ -4166,7 +4196,7 @@ private long SectionOffset(PdfDictionary trailer, PdfName key, long section, out // So does one leading to an object a guard cut, whose loading, once the chain is read, reaches the guard (#263). The // older sections, and their trailers, are the guard's to hide. - if (value is PdfReference && Follow(key, value, entry: true, out _, out var cut) is { } fault && cut is { } past) + if (value is PdfReference && Follow(key, value, entry: true, out _, out var cut, LengthLinks) is { } fault && cut is { } past) { _cutByGuard = true; _structure.TrailersHiddenByGuard = true; diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index f821ac34..77fd583d 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -1015,14 +1015,14 @@ private readonly DeclaredLength ReadLength(PdfDictionary dictionary) { return new DeclaredLength( StreamLengthForm.ObjectCut, - Reference: named.Target(), + Reference: named.Target(PdfReference.MaxChainLength), Hint: value.AsInteger() is { } hint and >= 0 and <= int.MaxValue ? (int)hint : null); } // The object a guard left out of the index, as the one it cut, is the one the chain of references leads to. if (presence == ObjectPresence.Unindexed) { - return new DeclaredLength(StreamLengthForm.ObjectUnindexed, Reference: named.Target()); + return new DeclaredLength(StreamLengthForm.ObjectUnindexed, Reference: named.Target(PdfReference.MaxChainLength)); } if (presence != ObjectPresence.Defined) diff --git a/src/AdCodicem.Pdf/IO/StreamLengthFault.cs b/src/AdCodicem.Pdf/IO/StreamLengthFault.cs index 72ccb482..6ca9a8d8 100644 --- a/src/AdCodicem.Pdf/IO/StreamLengthFault.cs +++ b/src/AdCodicem.Pdf/IO/StreamLengthFault.cs @@ -16,7 +16,10 @@ internal readonly record struct StreamLengthFault /// Gets how the /Length was written, or what the object it names holds. public required StreamLengthForm Form { get; init; } - /// Gets the object the /Length names, or null when it is written in the stream's dictionary. + /// + /// Gets the object the /Length names — for a length a guard cut or left out of the index, the one a chain of + /// references from it leads to, which the guard cut or left out —, or null when it is written in the stream's dictionary. + /// public PdfObjectId? Reference { get; init; } /// diff --git a/src/AdCodicem.Pdf/IO/StreamLengthForm.cs b/src/AdCodicem.Pdf/IO/StreamLengthForm.cs index 4934ece9..7613a9eb 100644 --- a/src/AdCodicem.Pdf/IO/StreamLengthForm.cs +++ b/src/AdCodicem.Pdf/IO/StreamLengthForm.cs @@ -32,8 +32,10 @@ internal enum StreamLengthForm : byte Deferred, /// - /// A reference to an object one of the reader's guards left out of the index: whether the file holds it is unknown. The - /// data is taken up to its endstream, and nothing is reported: the guard's report says why. + /// A reference to an object one of the reader's guards left out of the index, or that a chain of references from it + /// leads to: whether the file holds it is unknown. The data ends where its endstream ends it, or as a stream with + /// none ends; nothing is reported of the length, whose fault is the guard's, which reported it, and a missing + /// endstream is still the file's (#224). /// ObjectUnindexed, diff --git a/src/AdCodicem.Pdf/Objects/PdfReference.cs b/src/AdCodicem.Pdf/Objects/PdfReference.cs index 34d01204..fcfd1e15 100644 --- a/src/AdCodicem.Pdf/Objects/PdfReference.cs +++ b/src/AdCodicem.Pdf/Objects/PdfReference.cs @@ -52,13 +52,18 @@ public override PdfObject Resolve() /// Gives the object whose value gives: this reference's, or the last one a chain of objects whose /// values are references leads to, as far as follows it: a chain too long for it ends at its /// 31st object, the last whose value takes, so that no object past it is taken for the target. + /// It loads the objects before that one, which resolving the reference loads too. /// - internal PdfObjectId Target() + /// + /// How many objects the chain is followed through: one more for a value read from the object a reference names and then + /// resolved — a stream's /Length, a trailer's /Prev —, whose 32nd object gives it. + /// + internal PdfObjectId Target(int objects = MaxChainLength - 1) { var reference = this; // Resolve gives the value of a chain's 31st object at most: that object is the furthest a chain is followed to. - for (var depth = 1; depth < MaxChainLength - 1; depth++) + for (var depth = 1; depth < objects; depth++) { if (reference.Source?.GetObject(reference.Id) is not PdfReference next) { diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 013f75dc..1bd57681 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -2121,6 +2121,20 @@ public void A_deferred_length_s_data_with_no_endstream_before_the_next_object_ma $"its /Length is the reference 9 0 R, which no section read before it places where it can be read, and no endstream follows its data before the next object, at {next}")); } + [Fact] + public void A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to() + { + // Stream 7's /Length names object 5, which the newer table places, and which holds 6 0 R, which only stream 7 places: + // the length waits for the chain, then resolves to a name, which is said, as the parse says what it resolves to — not + // the reference object 5 holds (#220). + var file = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] /Length 5 0 R", "6 0 R", compressed: false, predicted: false, placed: true, six: "/Foo"); + using var document = PdfDocument.Open(file); + + document.Diagnostics.Should().ContainSingle().Which.Message.Should().Be( + "The /Length of cross-reference stream 7 names object 5 0, which holds the name /Foo, not a length; the chain took its data up to its endstream, after 56 bytes."); + } + [Fact] public void A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain() { diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 2480f907..7e3098c3 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -521,6 +521,126 @@ public void Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_de new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().BeFalse(); } + [Theory] + [InlineData(31)] + [InlineData(32)] + public void A_length_whose_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s(int objects) + { + // Content stream 4's /Length names object 100, which names the next, the last, after 3,000 spaces, holding 4: the + // parse resolves a /Length through 32 objects, one more than a reference, and the last it reads, cut, is the guard's + // — the 31st no longer taken for the target unread, nor the 32nd left past it (#224). + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + [4] = file.Object(4, "<< /Length 100 0 R >>\nstream\n0 0 m\nendstream"), + }; + var last = 100 + objects - 1; + + for (var number = 100; number <= last; number++) + { + rows[number] = file.Object(number, number < last ? Invariant($"{number + 1} 0 R") : new string(' ', 3000) + "4"); + } + + file.Table(last + 1, rows, Invariant($"/Size {last + 1} /Root 1 0 R")); + + using (var whole = PdfDocument.Open(file.Bytes)) + { + whole.GetObject(new PdfObjectId(4)).AsStream().Required().GetRawBytes().Length.Should().Be(5); + whole.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.StreamLengthInvalid); + } + + using var document = PdfDocument.Open(file.Bytes, ObjectGuard); + document.GetObject(new PdfObjectId(4)).AsStream().Required().GetRawBytes().Length.Should().Be(5); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.Reader.TryGetStreamLengthFault(4, out _).Should().BeFalse(); + } + + [Theory] + [InlineData(31)] + [InlineData(32)] + public void A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain(int objects) + { + // Stream 7's /Length names object 5, which the newer table places, and which names object 100, the next, and so on; + // the last, after 3,000 spaces, holds the length. The chain read follows as many objects as the parse resolves a + // /Length through, so the cut last one makes it wait for the chain, which then leaves it to the guard (#220, #224). + var placed = new Dictionary(); + var last = 100 + objects - 2; + + for (var number = 100; number <= last; number++) + { + placed[number] = number < last ? Invariant($"{number + 1} 0 R") : new string(' ', 3000) + "56"; + } + + var file = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] /Length 5 0 R", "100 0 R", compressed: false, predicted: false, placed: true, placedToo: placed); + + using (var whole = PdfDocument.Open(file)) + { + whole.Diagnostics.Should().BeEmpty(); + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.GetObject(new PdfObjectId(3)).AsDictionary().Should().NotBeNull(); + + var throwing = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); + throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); + } + + [Theory] + [InlineData(31)] + [InlineData(32)] + public void A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s(int objects) + { + // The newer table's /Prev names object 21, which names the next, the last, after 3,000 spaces, holding the older + // table's offset: a /Prev reads through 32 objects, as a whole read does, and the last, cut, ends the chain for the + // guard, which the 32nd did not: it was no offset, and the index was rebuilt (#263). + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 1 1] /Resources << >> >>"), + }; + var older = file.Table(4, new Dictionary(rows), "/Size 4 /Root 1 0 R"); + var newer = new Dictionary(); + var last = 21 + objects - 1; + + for (var number = 21; number <= last; number++) + { + newer[number] = file.Object( + number, number < last ? Invariant($"{number + 1} 0 R") : new string(' ', 3000) + older.ToString(CultureInfo.InvariantCulture)); + } + + var table = file.Position; + var text = new StringBuilder(Invariant($"xref\n0 1\n0000000000 65535 f \n21 {objects}\n")); + + foreach (var number in newer.Keys) + { + text.Append(CultureInfo.InvariantCulture, $"{newer[number]:D10} 00000 n \n"); + } + + text.Append(CultureInfo.InvariantCulture, $"trailer\n<< /Size {last + 1} /Root 1 0 R /Prev 21 0 R >>\nstartxref\n{table}\n%%EOF\n"); + file.Raw(text.ToString()); + + using (var whole = PdfDocument.Open(file.Bytes)) + { + whole.WasRepaired.Should().BeFalse(); + whole.Reader.Structure.Sections.Should().HaveCount(2); + } + + using var document = PdfDocument.Open(file.Bytes, ObjectGuard); + + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.Reader.Structure.Sections.Should().ContainSingle().Which.ChainHiddenByGuard.Should().NotBeNull(); + } + [Theory] [InlineData(false)] [InlineData(true)] @@ -646,4 +766,6 @@ public void A_parameter_a_guard_cut_before_an_image_filter_leaves_no_encoded_ima .Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); } } + + private static string Invariant(FormattableString text) => text.ToString(CultureInfo.InvariantCulture); } From 25bff0f8a28ffdb9c25b87ae88358e5e66cc451f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 11:49:15 +0000 Subject: [PATCH 043/146] fix: ask a larger window of a stream keyword a regular byte may go on Since the end-of-line after a stream keyword counts toward no window, the file was asked where the data starts when the window ended right after "stream". It skipped an end-of-line, and took any other byte for the data's first. "streamX" is no stream keyword: the dictionary before it stands alone, its object lacking its endobj, as every other window position read it. At that one position, the object read as a stream whose data started at the X. A regular byte after the keyword now leaves the decision to a larger window, as before the end-of-line was read from the file. Refs #174 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../IO/IPdfStreamDataProvider.cs | 4 +++ src/AdCodicem.Pdf/IO/PdfFileReader.cs | 5 +++ src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 9 +++-- tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs | 36 +++++++++++++++++++ 4 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs index 6dc82974..bd1dcfe7 100644 --- a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs +++ b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs @@ -25,6 +25,10 @@ internal interface IPdfStreamDataProvider /// return after it, and so cannot say: a window a guard ended there is not grown for it, since stream data counts /// toward no window (#174). /// + /// + /// Where the data starts, or -1 when a regular character follows the keyword, which makes it a longer one, as a larger + /// window says: the parser asks for one. + /// long DataStart(long absoluteKeywordEnd); /// diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index c2c85834..6cd54c31 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -1559,6 +1559,11 @@ long IPdfStreamDataProvider.DataStart(long absoluteKeywordEnd) var read = absoluteKeywordEnd < 0 || absoluteKeywordEnd >= _source.Length ? 0 : _source.Read(absoluteKeywordEnd, endOfLine[..(int)Math.Min(endOfLine.Length, _source.Length - absoluteKeywordEnd)]); + if (read > 0 && PdfCharacters.IsRegular(endOfLine[0])) + { + return -1; + } + var skipped = read > 0 && endOfLine[0] == (byte)'\r' ? 1 : 0; if (skipped < read && endOfLine[skipped] == (byte)'\n') diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 77fd583d..2ea5543f 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -708,16 +708,19 @@ private PdfStream ReadStream(PdfDictionary dictionary) // The end-of-line after "stream" is not data. A buffer that ends on the keyword, or on the carriage // return of a CR LF, cannot say where the data starts: the file says, as it says what follows the data, since the // data counts toward no window — not even one a guard ended there (#174) —; a parser the reader does not back - // needs a larger buffer. One that ends the data says: the stream's data, which starts there, holds nothing. + // needs a larger buffer, as does a keyword a regular character goes on, which is no stream keyword. One that ends + // the data says: the stream's data, which starts there, holds nothing. if (!_endsData && dataStart >= span.Length && (dataStart == keywordEnd || span[dataStart - 1] != (byte)'\n')) { - if (_streamData is null) + var start = _streamData?.DataStart(_baseOffset + keywordEnd) ?? -1; + + if (start < 0) { _truncated = true; } else { - dataStart = (int)(_streamData.DataStart(_baseOffset + keywordEnd) - _baseOffset); + dataStart = (int)(start - _baseOffset); } } diff --git a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs index cc637cfe..a6905231 100644 --- a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs +++ b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs @@ -182,6 +182,42 @@ public void A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword(st guarded.Diagnostics.Select(entry => entry.Message).Should().Equal(whole.Diagnostics.Select(entry => entry.Message)); } + [Theory] + [InlineData("X\nhello\nendstream")] + [InlineData("endstream")] + public void A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it(string tail) + { + // "streamX" is no stream keyword: the dictionary before it stands alone, its object lacking its endobj. A window that + // ends right after "stream" cannot say so, and the file is not asked where the data starts past a keyword a regular + // byte may go on: a larger window says (#174). + var failures = new List(); + + for (var shift = -2; shift <= 2; shift++) + { + var head = "4 0 obj\n<< /Length 5 /Pad () >>\nstream"; + var pad = Window + shift - head.Length; + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + [4] = file.Object(4, $"<< /Length 5 /Pad ({new string('x', pad)}) >>\nstream" + tail), + [6] = file.Object(6, "(six)"), + }; + file.Table(7, rows, "/Size 7 /Root 1 0 R"); + using var document = PdfDocument.Open(file.Bytes); + + var value = document.GetObject(new PdfObjectId(4)); + + if (value is not PdfDictionary || !document.Reader.IsEndObjMissing(4, out _)) + { + failures.Add($"shift {shift}: {value.GetType().Name}, endobj missing {document.Reader.IsEndObjMissing(4, out _)}"); + } + } + + failures.Should().BeEmpty(); + } + [Fact] public void A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword() { From 811f3bf716947b97ecfa6ac73bbc789cb78e1894 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 11:49:16 +0000 Subject: [PATCH 044/146] fix: leave a header a window's edge cuts to the edge only where one may begin A header the window's edge leaves undecided is the edge's, as round 1 settled. Two parts of it fell short: - the tokens read were not asked whether they rule a header out: a number 0, one past the object numbers, a generation past 65,535 and an object stream's member, which has none, all silenced the file's reports of the keys that are no names at a guard's cut; - inside a dictionary, the number of "6 0 obj" a guard cut at its keyword was still given to the key before it as its value, replacing the file's, with a repeat said to keep it. Only tokens that may still begin a header are left undecided now. Where a keyword that obj starts with follows two integers, which no reference can, the containers end there, as at a header: the key keeps what the file gave it, and nothing is said. "6 0" alone may be what the edge left of a reference, and is kept as far as it was read, as before. The parser test of "[1 2\n6 0 obj" at the edge counted the header's tokens as the array's elements; it now counts the two before it, as the rule reads it, still asking for a larger window. Refs #230 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 50 +++++++++++++++---- tests/AdCodicem.Pdf.Tests/ParserTests.cs | 21 +++++++- tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs | 50 +++++++++++++++++++ 3 files changed, 110 insertions(+), 11 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 2ea5543f..381ba128 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -68,6 +68,13 @@ internal ref struct PdfObjectParser /// private int _headerAt = -1; + /// + /// Whether the header says is one a window's edge left undecided, which a larger window may show + /// is none: what the edge made of it, which ends the containers as a header does, but reports nothing and gives the key + /// before it no value (#230). + /// + private bool _headerUndecided; + /// /// Whether the last stream read kept a declared length that neither endstream nor endobj follows: where /// its object ends is unknown, and its endobj is not judged. @@ -204,6 +211,7 @@ public int Position public PdfObject ParseObject() { _headerAt = -1; + _headerUndecided = false; return ParseValue(_lexer.Read(), 0); } @@ -269,6 +277,7 @@ public bool TryReadIndirectObject(out PdfObjectId id, out PdfObject value) _objectNumber = id.Number; _headerStart = number.Start; _headerAt = -1; + _headerUndecided = false; // An empty object, "2 0 obj endobj", reads its endobj as its value, and says so; that endobj is its own. var beforeValue = _lexer.Position; @@ -430,9 +439,22 @@ private PdfObject ParseIntegerOrReference(PdfToken token, int depth) return PdfNull.Instance; } - // The buffer ended before the look-ahead could tell: "12 0" may be all there is, or what a - // window's edge left of "12 0 R" — or, inside a container, of the next object's header, as "12 0 ob" may be. - if (third.Kind == PdfTokenKind.EndOfInput || (depth > 0 && HeaderUndecided(second, third))) + // Inside a container, a window's edge may have cut a keyword that obj starts with after two integers, "12 0 ob": no + // reference, and maybe the next object's header, which a larger window shows. The containers end there, as at a + // header, and the number is no value of the file's. "12 0" alone may be what the edge left of a reference, and is + // kept as far as it was read, as any value the edge cuts. + if (depth > 0 && third.Kind == PdfTokenKind.Keyword && HeaderUndecided(token, second, third)) + { + _truncated = true; + _headerAt = token.Start; + _headerUndecided = true; + _lexer.Position = saved; + return PdfNull.Instance; + } + + // The buffer ended before the look-ahead could tell: "12 0" may be all there is, or what a window's edge left of + // "12 0 R". + if (third.Kind == PdfTokenKind.EndOfInput) { _truncated = true; } @@ -494,7 +516,10 @@ private PdfObject ParseArray(int depth, int openedAt) // array or a dictionary the element opened, reported there, and that element kept as it was read. if (_headerAt == token.Start) { - ReportCut("an array", openedAt, depth, atHeader: true); + if (!_headerUndecided) + { + ReportCut("an array", openedAt, depth, atHeader: true); + } } else { @@ -629,7 +654,12 @@ private PdfObject ParseDictionaryOrStream(int depth, int openedAt) { // The next object's header ends the dictionary: where the value should be, the key given none and reported // here; inside an array or a dictionary the value opened, reported there, and that value kept. - if (_headerAt == valueToken.Start) + if (_headerAt == valueToken.Start && _headerUndecided) + { + // One a window's edge left undecided is the edge's: the key keeps what it held, and nothing is said. + Give(dictionary, ref nullKeys, keyToken, key, PdfNull.Instance, keyAtEdge: true, valueCut: true); + } + else if (_headerAt == valueToken.Start) { ReportCut("a dictionary", openedAt, depth, valueMissing: true, atHeader: true); Give(dictionary, ref nullKeys, keyToken, key, PdfNull.Instance, keyAtEdge, valueCut: false); @@ -1442,7 +1472,7 @@ private bool StartsHeader(PdfToken number) // Tokens a window's edge leaves undecided may be a header a larger window shows: the parse met the edge, and what it // reads from here — the number and the generation, as keys — is no fault of the file's. - if (HeaderUndecided(generation, keyword)) + if (HeaderUndecided(number, generation, keyword)) { _truncated = true; } @@ -1452,10 +1482,12 @@ private bool StartsHeader(PdfToken number) /// /// Determines whether a window's edge leaves undecided whether a number and the two tokens after it are the next - /// object's header: the edge comes before a generation, before a keyword, or inside one that obj starts with. + /// object's header: the edge comes before a generation, before a keyword, or inside one that obj starts with, + /// and no whole token read rules a header out — a number that is no object number, a generation that is none + /// (). An object stream's member is followed by no header. /// - private readonly bool HeaderUndecided(PdfToken generation, PdfToken keyword) => - !_endsData && + private readonly bool HeaderUndecided(PdfToken number, PdfToken generation, PdfToken keyword) => + !_endsData && _member is null && MayBeginHeader(number, generation, _memory.Length) && (generation.Kind == PdfTokenKind.EndOfInput || (generation.Kind == PdfTokenKind.Integer && (keyword.Kind == PdfTokenKind.EndOfInput || diff --git a/tests/AdCodicem.Pdf.Tests/ParserTests.cs b/tests/AdCodicem.Pdf.Tests/ParserTests.cs index 2ce37743..091c3e93 100644 --- a/tests/AdCodicem.Pdf.Tests/ParserTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ParserTests.cs @@ -255,14 +255,14 @@ public void Tokens_a_window_s_edge_leaves_undecided_as_a_header_are_no_stray_key } [Theory] - [InlineData("[1 2\n6 0 obj", true, 5)] + [InlineData("[1 2\n6 0 obj", true, 2)] [InlineData("[1 2\n6 0 obj ", false, 2)] [InlineData("[1 2\n6 0 obj\n", false, 2)] public void Takes_an_object_header_a_window_s_edge_reaches_for_none_until_a_larger_window_shows_obj_whole( string text, bool more, int count) { // "obj" the edge reaches may be the start of "objx", no keyword at all: what the window reads is dropped for a larger - // window, which says (#230). + // window, which says (#230). The window ends the array where the header may start, its number no element. var parser = new PdfObjectParser(Encoding.ASCII.GetBytes(text), endsData: false); parser.ParseObject().AsArray().Required().Should().HaveCount(count); @@ -761,6 +761,23 @@ public void Keeps_what_the_end_of_the_data_cut_short_as_far_as_it_was_read() hexadecimal.Bytes.ToArray().Should().Equal((byte)0x41, 0x42, 0x40); } + [Theory] + [InlineData("5 0 << /A 6 0 o")] + [InlineData("5 0 << /A 6 0 ob")] + public void Reports_the_keys_of_a_member_a_window_s_edge_leaves_as_a_header_none_can_be(string text) + { + // An object stream's member has no header after it: its window's edge leaves none undecided, and the key 0 that is no + // name is the file's, as in the whole member (#230). + var diagnostics = new PdfDiagnostics(); + var parser = PdfObjectParser.ForObjectStreamMember( + Encoding.ASCII.GetBytes(text), 9, 1000, 5, ObjectSourceReturning(new PdfObjectId(1), PdfNull.Instance), diagnostics, endsData: false); + parser.Position = 4; + + _ = parser.ParseObject(); + + diagnostics.Should().Contain(entry => entry.Code == PdfDiagnosticCodes.SyntaxUnexpectedToken); + } + [Fact] public void Says_an_object_stream_s_data_rather_than_the_file_ended_inside_a_member_s_construct() { diff --git a/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs b/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs index 7375496a..7793483b 100644 --- a/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs +++ b/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs @@ -303,6 +303,56 @@ public void A_header_the_guard_s_cut_leaves_undecided_is_the_guard_s(string tail document.Diagnostics.Should().Contain(entry => entry.Code == PdfDiagnosticCodes.LimitObject); } + [Fact] + public void A_header_the_guard_s_cut_leaves_undecided_gives_the_key_before_it_no_value() + { + // "6" after the second /A may be the value it is given, or the next object's header, which a larger limit reads, the + // key then left out: the guard's cut decides neither, the key keeps the value the file gave first, and the repeat + // is not said to keep a last one (#230). + var document = CutAt(" /A 1 /A\n\n6 0 obj", 128); + + document.GetObject(new PdfObjectId(4)).AsDictionary().Required().GetInteger(PdfName.Get("A")).Should().Be(1); + document.Diagnostics.Should().NotContain(entry => entry.Code == PdfDiagnosticCodes.SyntaxKeyRepeated); + document.Dispose(); + } + + [Theory] + [InlineData(" /A 1\n\n0 0 obj", 2)] + [InlineData(" /A 1 /B 6 70000 obj", 1)] + public void Tokens_no_header_begins_with_are_reported_at_the_guard_s_cut_as_past_it(string tail, int keys) + { + // An object number 0, or a generation past 65,535, rules a header out whatever the cut hides: the keys that are not + // names are the file's, reported as under a larger limit (#175, #230). + using var guarded = CutAt(tail, 128); + using var whole = CutAt(tail, 100_000); + guarded.GetObject(new PdfObjectId(4)); + whole.GetObject(new PdfObjectId(4)); + + var reported = guarded.Diagnostics.Where(entry => entry.Code == PdfDiagnosticCodes.SyntaxUnexpectedToken).Select(entry => entry.Position).ToArray(); + reported.Should().HaveCount(keys); + whole.Diagnostics.Where(entry => entry.Code == PdfDiagnosticCodes.SyntaxUnexpectedToken).Select(entry => entry.Position).Should().Contain(reported); + } + + /// + /// Opens a document whose object 4 lost its >> and its endobj, its padding setting 's + /// end where a MaxObjectLength of 128 cuts it, under . + /// + private static PdfDocument CutAt(string tail, int limit) + { + const int Edge = 128; + var head = "4 0 obj\n<< /Pad ("; + var body = "<< /Pad (" + new string('x', Edge - head.Length - (")" + tail).Length) + ")" + tail + "\n<< /Foo 7 >>"; + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + [4] = file.Object(4, body), + }; + file.Table(5, rows, "/Size 5 /Root 1 0 R"); + return PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = limit } }); + } + /// /// A sound document whose catalog names objects 4 and 5, object 4 holding and no endobj, then /// object 5 holding and its endobj. From 7cce393d85a99e1b49b827d258fad9c1f5abed55 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 11:53:17 +0000 Subject: [PATCH 045/146] fix: take no endobj a window's edge reaches for whole An object's value read, the token after it was taken for its endobj whenever it spelled the keyword, though one a window's edge reaches may be the start of a longer one. "endobjx" is no endobj: the object lacks it, as every other window position read it. At the one position where "endobj" ended at the edge, the object was said to have it, and what the file read on after a stream's endstream said otherwise was set aside. The token is now taken whole only as the containers take it: a stream's object keeps what the file read on after its endstream says, and any other value asks for a larger window. The gap is older than the batch, which made the reading of what follows a stream's end the same at every window position. Refs #174 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 8 ++-- tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs | 41 ++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 381ba128..0e721531 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -298,15 +298,17 @@ public bool TryReadIndirectObject(out PdfObjectId id, out PdfObject value) var unseen = next.Kind == PdfTokenKind.EndOfInput || next.End >= _memory.Length; // What follows a stream whose data runs past the buffer lies past it too: the file was asked at the stream's - // end, and its answer stands where the buffer has none (#55). - _endObj = next.IsKeyword("endobj"u8) + // end, and its answer stands where the buffer has none (#55). An endobj a window's edge reaches may be the start of + // a longer keyword, which the file read on after a stream says, or a larger window (#174). + var endObj = IsEndObj(next); + _endObj = endObj ? EndObjState.Present : _endObjUnknown ? EndObjState.Unknown : !unseen ? EndObjState.Absent : value is PdfStream && _endObjAfterStream is { } afterStream ? afterStream : EndObjState.Unseen; - if (!next.IsKeyword("endobj"u8)) + if (!endObj) { // A value followed by the end of the buffer, or by a token the end of the buffer may have cut, // may itself have been cut there: a string that lost its closing parenthesis, a dictionary whose diff --git a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs index a6905231..e2763586 100644 --- a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs +++ b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs @@ -218,6 +218,47 @@ public void A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge failures.Should().BeEmpty(); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public void An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it(bool stream) + { + // "endobjx" is no endobj: the object lacks it. One the window's edge reaches may be the start of a longer keyword, + // and is not taken whole: a stream's, as the file read on after its endstream says, and any other value's, as a larger + // window says (#174). + var failures = new List(); + + for (var shift = -2; shift <= 2; shift++) + { + // The data's length that ends the keyword "4 0 obj\n" and the body leave at the edge, shifted. + var body = Enumerable.Range(0, Window) + .Select(data => stream + ? string.Create(CultureInfo.InvariantCulture, $"<< /Length {data} >>\nstream\n") + new string('a', data) + "\nendstream\nendobj" + : "(" + new string('a', data) + ")\nendobj") + .First(text => "4 0 obj\n".Length + text.Length == Window + shift); + + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + [4] = file.Object(4, body + "x"), + [6] = file.Object(6, "(six)"), + }; + file.Table(7, rows, "/Size 7 /Root 1 0 R"); + using var document = PdfDocument.Open(file.Bytes); + + document.GetObject(new PdfObjectId(4)); + + if (!document.Reader.IsEndObjMissing(4, out _)) + { + failures.Add(string.Create(CultureInfo.InvariantCulture, $"shift {shift}: endobj taken whole")); + } + } + + failures.Should().BeEmpty(); + } + [Fact] public void A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword() { From d134848417f741b0a29dab2c16e70202f436fb6b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 11:54:02 +0000 Subject: [PATCH 046/146] docs: say where decoding ends at a value a guard cut, and where that is reported Decode's remarks named the end of the chain past MaxFilterChainLength, but not the one a /Filter, a /DecodeParms or a parameter that leads to an object a guard cut makes. The data is left as the filters before it decoded it, and the guard's report lies in the document's diagnostics, as everything loading a value meets does. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs b/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs index 8b93ec05..2056d6d6 100644 --- a/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs +++ b/src/AdCodicem.Pdf/Objects/PdfStreamDecodingExtensions.cs @@ -51,7 +51,10 @@ public static class PdfStreamDecodingExtensions /// own each time the stream is decoded. A /Filter array that /// names more filters than /// ends decoding before the first past them, reported as - /// when decoding reaches it. + /// when decoding reaches it. A /Filter, a /DecodeParms + /// or a parameter that leads to an object one of the document's guards cut, or left out of the index, ends decoding + /// the same way, at the filter it names or whose parameters it gives: the guard was reported in the document's + /// diagnostics where it was reached, as everything loading a value meets is, and nothing more is reported. /// /// /// The filters and the values of their parameters are read before any of the data is: a parameter held in an From 5b30aa18ae2e1503e801bf007d9ba7807f424fcc Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 12:02:12 +0000 Subject: [PATCH 047/146] test: hold the copy of a hidden chain on relocation and the check kept for a second reading No test failed with the copy of what a guard-cut /Prev hides left out of a section's relocation: every such section was read where it was named. One is now found 3 bytes from there, and the validation still says what it hides. The test named for keeping, for a stream's second reading, what the white space after its declared length leads to held only what both readings give. It now counts the bytes the second reading reads: no more than its window and the first read after its data. Refs #174, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 25 +++++++++++++++++++ .../AdCodicem.Pdf.Tests/StreamLengthTests.cs | 7 ++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 7e3098c3..698215f7 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -4,6 +4,7 @@ using AdCodicem.Pdf.Diagnostics; using AdCodicem.Pdf.Documents; using AdCodicem.Pdf.IO; +using AdCodicem.Pdf.IO.XRef; using AdCodicem.Pdf.Objects; using AdCodicem.Pdf.Validation; @@ -700,6 +701,30 @@ public void A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s(string ke throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); } + [Fact] + public void A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved() + { + // A third table's /Prev names the table whose /Prev, object 9, MaxObjectLength cuts, 3 bytes before it: the section is + // found where it lies, and what its /Prev hides is said of it there, as where it is named (#263). + var two = ChainFiles.UpdateNamingItsPrevious("9 0 R", padding: 3000); + var text = Encoding.Latin1.GetString(two); + var named = long.Parse(text[(text.LastIndexOf("startxref\n", StringComparison.Ordinal) + "startxref\n".Length)..].Split('\n')[0], CultureInfo.InvariantCulture); + var five = two.Length; + var appended = Invariant($"5 0 obj\n(five)\nendobj\n"); + var third = five + appended.Length; + appended += Invariant( + $"xref\n0 1\n0000000000 65535 f\r\n5 1\n{five:D10} 00000 n\r\ntrailer\n<< /Size 10 /Root 1 0 R /Prev {named - 3} >>\nstartxref\n{third}\n%%EOF\n"); + var file = two.Concat(Encoding.Latin1.GetBytes(appended)).ToArray(); + + using var document = PdfDocument.Open(file, ObjectGuard); + + var moved = document.Reader.Structure.Sections.Should().ContainSingle(section => section.State == XRefSectionState.Relocated).Which; + moved.ChainHiddenByGuard.Should().NotBeNull(); + new PdfValidator().Validate(document).Findings + .Where(finding => finding.RuleId == PdfValidationRuleIds.XRefCheckedInPart) + .Should().ContainSingle(finding => finding.Message.Contains("names older sections", StringComparison.Ordinal)); + } + [Fact] public void A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain() { diff --git a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs index 23a2542e..6c0da9aa 100644 --- a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs +++ b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs @@ -2227,7 +2227,7 @@ public void A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_ch { // The declared length ends 20 bytes short of the window's edge, and a comment after it runs past the edge and past the // first read after it: the reader reads on through it to the endstream, and keeps what it found for the stream's next - // reading, after the cache let the stream go. + // reading, after the cache let the stream go, which reads nothing past its window and the first read after its data. var file = new HandWrittenFile(); var rows = new Dictionary { @@ -2247,7 +2247,8 @@ public void A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_ch } file.Table(80, rows, "/Size 80 /Root 1 0 R"); - using var document = PdfDocument.Open(file.Bytes, PdfReaderOptions.Default with { ObjectCacheCapacity = 64 }); + using var source = new StrictCountingSource(file.Bytes); + using var document = PdfDocument.Open(source, PdfReaderOptions.Default with { ObjectCacheCapacity = 64 }, ownsSource: false); document.GetObject(new PdfObjectId(4)).AsStream().Required().RawLength.Should().Be(length); @@ -2256,7 +2257,9 @@ public void A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_ch document.GetObject(new PdfObjectId(number)); } + var before = source.BytesRead; document.GetObject(new PdfObjectId(4)).AsStream().Required().RawLength.Should().Be(length); + (source.BytesRead - before).Should().BeLessThanOrEqualTo(PdfFileReader.InitialObjectWindow + PdfObjectParser.EndObjLookahead); document.Diagnostics.Should().BeEmpty(); document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); } From 80e85519122b6ed6b8120e0ff3d3f8563e0c01a2 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 12:02:13 +0000 Subject: [PATCH 048/146] docs: say which code a stream taken without a search earns, and what a header cut leaves stream.truncated said its data may run to the end of the file, which data taken without a search does under stream.length-invalid: no search said its endstream is missing. Both codes, and the reference page, now say so. The threat model and the concepts page said a header the window's edge cuts short is never taken for a value. That holds of "6 0 ob", which no reference can be; "6 0" alone is kept as far as it was read, as what the edge may have left of a reference. Tokens that rule a header out are reported. The routing table of Decode's diagnostics says what loading a value the filters read meets is the document's alone. Refs #170, #174, #230 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 2 +- docs/website/docs/concepts/diagnostics.md | 2 +- docs/website/docs/reference/diagnostics.md | 4 ++-- src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs | 8 +++++--- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index dc2a0798..0a20890c 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -248,7 +248,7 @@ Known gaps: | The lexer looping, or reading past its buffer | Every token but the end of input consumes a byte. Look-aheads check the length, and a string that never closes ends at the buffer's end, and is told from a closed one by its token's length | `PropertyTests.The_lexer_terminates_on_any_bytes_and_stays_inside_the_buffer`; `LexerTests.Never_loops_on_a_stray_delimiter`, `Recovers_from_an_unterminated_literal_string`; `ParserTests.Reports_nothing_of_a_construct_closed_at_the_end_of_the_data` | Property, over buffers of about 100 bytes | | Nesting that exhausts the stack | `PdfObjectParser.MaxDepth` (128) and a stack probe at each container. Past either, `SkipContainer` counts brackets without recursion, and `syntax.depth-exceeded` is reported | `ParserTests.Refuses_to_follow_nesting_deeper_than_its_limit`; `HostileInputTests.Survives_deeply_nested_containers_inside_an_object`, `A_file_nesting_objects_deeper_than_the_stack_allows_reads_them_as_null_without_crashing` | No | | A token or an object of any length, or one that never closes | The window grows ×8 from 8 KB up to `MaxObjectLength`, and the parser is told whether its window ends the data: only then is what it leaves open reported, once, `syntax.truncated-object` where the innermost construct opens, and a window that ends where the file does cuts nothing however long it is. Past the bound, the object is kept as far as it was read and `limit.object` is reported, beside what the parse met before the cut; nothing is reported of the token the cut reaches, nor of what follows once a look-ahead met it (ADR 34). An array or a dictionary an `endobj` finds open ends there, rather than take the objects after it, and one that lost its own `endobj` too ends at the next object's header (below). The parses of one object add up to about 1.3 times the last window | `HostileInputTests.Survives_a_string_longer_than_the_largest_window`, `A_string_that_never_closes_takes_the_end_of_the_file_and_is_reported_once_where_it_opens`; `ReaderLimitsTests.An_object_is_read_no_further_than_its_bound`, `An_object_cut_by_its_bound_is_not_reported_as_damage`, `An_object_its_bound_cuts_anywhere_in_its_last_tokens_reports_the_bound_alone`, `What_an_object_its_bound_cuts_met_before_the_cut_is_still_reported`, `A_fault_after_a_stream_whose_length_the_file_cannot_hold_is_still_reported_beside_the_bound`, `An_object_its_bound_cuts_after_more_faults_than_the_diagnostics_hold_still_reports_the_bound`, `A_number_its_bound_cuts_is_no_number_out_of_range`, `An_object_as_long_as_its_bound_that_ends_the_file_is_read_whole`; `WindowEdgeTests.An_anomaly_inside_an_object_longer_than_the_window_is_reported_once`; `PropertyTests.What_a_window_reports_of_its_syntax_the_whole_data_reports_too_in_the_same_order`, `A_buffer_that_ends_at_a_window_s_edge_reports_nothing_of_the_end_of_the_data`; `SyntaxFaultTests.A_construct_the_file_ends_inside_is_reported_where_it_opens`, `A_container_an_endobj_ends_takes_nothing_of_the_objects_after_it`, `A_fault_of_an_object_read_through_a_window_the_file_goes_on_past_is_reported`, `A_rebuild_reports_no_fault_where_its_window_cuts_a_trailer`, `A_trailer_a_source_cut_short_ends_inside_is_reported_as_the_end_of_the_data`; `FileRuleTests.A_trailer_whose_closing_brackets_end_its_table_s_window_is_judged_by_what_they_close` | Growth yes, the bound no; the window's edge by property | -| An object that lost both its closing delimiter and its `endobj` taking in the next object, up to that object's `endobj`, its own missing `endobj` unreported; a trailer that lost its `>>` taking the next update's entries; a header the window's edge cut short reported as the file's stray keys | Inside an array or a dictionary, where an element, a key or a value is expected, the next object's header ends every container left open: `N G obj`, digits alone, `N` in 1..`int.MaxValue` and `G` in 0..65,535, white space alone between them, and `obj` a whole keyword the window's edge does not reach — the header the search for a lost `endstream` stops at (`HeaderText`). A header the edge cuts short, `6 0` or `6 0 ob`, is the edge's, as a reference it cuts is: where no larger window follows — at `MaxObjectLength`, in the rebuild's trailer scan —, its numbers are neither reported as keys that are not names nor taken for the value of the key before them. It is reported once, at the innermost container, as `syntax.truncated-object`, "An object header ended the object inside …", or, in a trailer or a value read alone, where no object is open, "An object header came inside …", and the object lacks its `endobj`. Containers `SkipContainer` passes over past `MaxDepth` end there too, and a trailer is read so as well. An object stream's member, which no header follows, and a value alone, which is no container, do not apply it: in `4 0 obj 5 0 obj`, object 4 holds the integer 5. Its cost, accepted: a closed container holding text that reads as a header ends there, and its `endobj` is judged missing; none of the 344,361 objects of the corpus holds one ([#230]) | `SyntaxFaultTests.An_object_that_lost_its_closing_delimiter_and_its_endobj_ends_at_the_next_object_s_header`, `The_next_object_s_header_ends_every_container_left_open_and_is_reported_at_the_innermost`, `Text_that_is_no_object_header_ends_no_container`, `A_header_inside_a_closed_container_ends_it_there_and_its_endobj_is_judged_missing`, `An_object_stream_member_holding_a_header_reads_it_as_tokens`, `Nesting_deeper_than_the_reader_follows_ends_at_the_next_object_s_header_too`, `A_trailer_that_lost_its_closing_delimiter_takes_nothing_of_the_update_after_it`, `A_header_the_guard_s_cut_leaves_undecided_is_the_guard_s`; `ParserTests.Ends_the_containers_an_object_left_open_at_the_next_object_s_header_and_leaves_it_to_be_read`, `Takes_an_object_header_a_window_s_edge_reaches_for_none_until_a_larger_window_shows_obj_whole`, `A_header_where_a_key_or_a_value_is_expected_ends_the_dictionary`, `A_value_alone_that_is_a_header_s_number_is_an_integer`, `Tokens_a_window_s_edge_leaves_undecided_as_a_header_are_no_stray_keys`; `WindowEdgeTests.An_object_that_lost_its_end_ends_at_the_next_header_wherever_the_window_edge_cuts_it` | Plausibly | +| An object that lost both its closing delimiter and its `endobj` taking in the next object, up to that object's `endobj`, its own missing `endobj` unreported; a trailer that lost its `>>` taking the next update's entries; a header the window's edge cut short reported as the file's stray keys | Inside an array or a dictionary, where an element, a key or a value is expected, the next object's header ends every container left open: `N G obj`, digits alone, `N` in 1..`int.MaxValue` and `G` in 0..65,535, white space alone between them, and `obj` a whole keyword the window's edge does not reach — the header the search for a lost `endstream` stops at (`HeaderText`). A header the edge cuts short, `6 0` or `6 0 ob`, is the edge's, as a reference it cuts is: where no larger window follows — at `MaxObjectLength`, in the rebuild's trailer scan —, its numbers are not reported as keys that are not names, unless a whole token already rules a header out — a number 0, a generation past 65,535, a member, which no header follows —; `6 0 ob`, a keyword `obj` starts after two integers, which no reference can be, ends the containers as a header does, its number given to no key and nothing reported, while `6 0` alone, what the edge may have left of a reference, is kept as far as it was read, as any value the edge cuts. It is reported once, at the innermost container, as `syntax.truncated-object`, "An object header ended the object inside …", or, in a trailer or a value read alone, where no object is open, "An object header came inside …", and the object lacks its `endobj`. Containers `SkipContainer` passes over past `MaxDepth` end there too, and a trailer is read so as well. An object stream's member, which no header follows, and a value alone, which is no container, do not apply it: in `4 0 obj 5 0 obj`, object 4 holds the integer 5. Its cost, accepted: a closed container holding text that reads as a header ends there, and its `endobj` is judged missing; none of the 344,361 objects of the corpus holds one ([#230]) | `SyntaxFaultTests.An_object_that_lost_its_closing_delimiter_and_its_endobj_ends_at_the_next_object_s_header`, `The_next_object_s_header_ends_every_container_left_open_and_is_reported_at_the_innermost`, `Text_that_is_no_object_header_ends_no_container`, `A_header_inside_a_closed_container_ends_it_there_and_its_endobj_is_judged_missing`, `An_object_stream_member_holding_a_header_reads_it_as_tokens`, `Nesting_deeper_than_the_reader_follows_ends_at_the_next_object_s_header_too`, `A_trailer_that_lost_its_closing_delimiter_takes_nothing_of_the_update_after_it`, `A_header_the_guard_s_cut_leaves_undecided_is_the_guard_s`; `ParserTests.Ends_the_containers_an_object_left_open_at_the_next_object_s_header_and_leaves_it_to_be_read`, `Takes_an_object_header_a_window_s_edge_reaches_for_none_until_a_larger_window_shows_obj_whole`, `A_header_where_a_key_or_a_value_is_expected_ends_the_dictionary`, `A_value_alone_that_is_a_header_s_number_is_an_integer`, `Tokens_a_window_s_edge_leaves_undecided_as_a_header_are_no_stray_keys`; `WindowEdgeTests.An_object_that_lost_its_end_ends_at_the_next_header_wherever_the_window_edge_cuts_it` | Plausibly | | Look-ahead that makes parsing superlinear | Look-ahead is a constant number of tokens, so each token is lexed three times at most | `ParserTests.Distinguishes_a_reference_from_two_integers`, `Backtracks_when_two_integers_are_not_followed_by_R` | Property | | A number past a `long` or a `double` | The number parser never wraps: an integer past a `long` reads as the real nearest to it, and a number past a double's range as an infinity, which the object parser reads as null under `syntax.number-out-of-range`, quoting the number only in a report the diagnostics keep. A real reads as the double nearest to its decimal. The shape — signs, then digits around at most one period — is checked before the framework's parse, which would read `NaN` and `Infinity`; each digit is read twice at most, and nothing is allocated | `LexerTests.Reads_an_integer_too_large_for_a_long_as_a_real`, `Reads_a_number_past_the_largest_double_as_an_infinity_its_reader_can_tell`, `Parsing_a_number_allocates_nothing_however_long_it_is`; `PropertyTests.The_integer_parser_reads_every_long_and_never_wraps_past_one`, `The_real_parser_reads_the_double_nearest_to_the_decimal`; `ParserTests.Reads_a_number_past_the_largest_real_as_null_and_reports_it`, `Reports_no_number_its_double_holds_however_it_rounds`; `HostileInputTests.Reads_a_number_fifteen_million_digits_long_in_time`, `Quotes_no_number_beyond_what_a_real_can_hold_once_the_diagnostics_are_full` | Property | | An object number or a generation overflowing its cast | The parser builds a header only from a number in 1..`int.MaxValue`, and a reference only from one in 0..`int.MaxValue` — object 0 heads the free list and reads as null —, each with a generation in 0..65,535 (`PdfObjectId.MaxNumber` and `MaxGeneration`, with their reasons), checked before the cast. The index and object streams check theirs the same way, before narrowing: see *Indexing* and *Resolving objects and object streams* | `ParserTests.Refuses_an_object_header_whose_number_or_generation_is_out_of_range`, `Makes_no_reference_of_numbers_an_object_identifier_cannot_hold`; `HostileInputTests.Serves_no_object_under_a_number_past_a_long_that_once_wrapped_to_its_number` | No | diff --git a/docs/website/docs/concepts/diagnostics.md b/docs/website/docs/concepts/diagnostics.md index 7a31c890..2433bbc8 100644 --- a/docs/website/docs/concepts/diagnostics.md +++ b/docs/website/docs/concepts/diagnostics.md @@ -31,7 +31,7 @@ happened to end. When `MaxObjectLength` allows no larger window, the object is k guard is reported: what the parse met before the edge — a stray token, a key that is not a name — is reported beside it, and nothing of the token the edge cut, nor of what that token leaves open, nor of the next object's header the edge cut short inside an array or a dictionary — `6 0` or `6 0 ob`, whose number and generation would otherwise read as keys -that are not names, or the number as a value —, is. A stream's data is no part of what a window must hold: where it +that are not names, or, for `6 0 ob`, which no reference can be, the number as a value —, is. A stream's data is no part of what a window must hold: where it ends is found in the file, whatever window its dictionary was read through, and it never counts toward `MaxObjectLength`. diff --git a/docs/website/docs/reference/diagnostics.md b/docs/website/docs/reference/diagnostics.md index 91173e4c..c886048a 100644 --- a/docs/website/docs/reference/diagnostics.md +++ b/docs/website/docs/reference/diagnostics.md @@ -74,7 +74,7 @@ The list `PdfDocument.Diagnostics` holds, in the order the entries were recorded | `Decode(stream, diagnostics)` | Where the reports go | |---|---| -| `diagnostics` given | Into it, each time the stream is decoded; the document's own `Diagnostics` also hold, once, the `limit.stream-length` the reader reports as it reads data `MaxStreamLength` cuts | +| `diagnostics` given | Into it, each time the stream is decoded; the document's own `Diagnostics` also hold, once, the `limit.stream-length` the reader reports as it reads data `MaxStreamLength` cuts. What loading a value the filters read meets — the `/Filter` entry, a filter's parameters — is the document's alone, a guard that cut it included, which ends decoding there | | none, a stream read from a document | The document's own `Diagnostics` | | none, a stream built in memory | Nowhere: nothing is reported | @@ -106,7 +106,7 @@ Codes are stable: they are part of the public contract, because callers filter o | `xref.chain-cycle` | The chain of previous sections looped: a `/Prev` or `/XRefStm` names an offset the chain has already named, whether it read a section there or not. Reported at that offset, or, when it lies outside the file, at the section whose trailer named it | | `xref.entry-out-of-range` | An object's entry places it outside the file — past its end, before its start, or past what a `long` holds once the header's offset is added —; reported with no position, the offset in the message, exactly. A section named outside the file is `xref.section-missing`'s, or the rebuild's when `startxref` names it | | `stream.length-invalid` | A stream's `/Length` is not where its data ends: its `endstream` lies elsewhere and ends the data; or the `/Length` is no length — absent, not a non-negative integer, or naming an object the file lacks or that could not be read — and the `endstream` ends the data; or no `endstream` lies before the next object or the end of the file, no `endobj` of its object ends the data, and the data is the declared length, when it ends before that stop, or runs up to the next object; or the document's searches for `endstream` have read as much of the file as they may, and the data is taken without a search. A `/Length` that names an object a guard cut, or left out of the index, is not judged, though a missing `endstream` is reported all the same. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case. A cross-reference stream whose `/Length` the chain could not read, its data taken up to its first `endstream`, has it read once the chain is, unless a guard cut that data: a `/Length` that is no length, or that no `endstream` confirms, is reported as any stream's, and one an `endstream` past the one the chain stopped at confirms is reported with the rows the chain left unread. Where the section's own row for that `/Length` lies among the rows the data holds, and gives a length an `endstream` confirms, the data is taken again to it before the rows are read, and nothing is reported; nor is anything of a `/Length` a guard cut or left out of the index | -| `stream.truncated` | A stream has no `endstream` before the next object or the end of the file, and its object's `endobj` ends its data — following the declared length, which is kept, or ending the object before that stop —; or neither, no declared length ends before the end of the file, and its data runs to it. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case | +| `stream.truncated` | A stream has no `endstream` before the next object or the end of the file, and its object's `endobj` ends its data — following the declared length, which is kept, or ending the object before that stop —; or neither, no declared length ends before the end of the file, and its data runs to it. Data taken without a search, the document's searches having read as much of the file as they may, is `stream.length-invalid`'s, to the end of the file too. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case | | `stream.self-reference` | An object stream's dictionary names an object the stream holds — as its `/Type`, `/Length`, `/N`, `/First`, a filter or a parameter —: that object reads as null while the stream is decoded, and the stream is decoded without it | | `object-stream.member-moved` | An object is in the object stream its entry names, at another index than the entry gives, and was read where the stream's header lists it. Reported where the stream's data starts, the object, the stream and both indexes in the message | | `object-stream.unreadable` | What an object stream — a stream an entry of the index names as one — says of itself cannot be believed: it is not of `/Type /ObjStm`, its `/N` is absent or no non-negative integer, or, when it declares objects, its `/First` is, or lies past its decoded data, or its header ends or breaks before listing as many objects as `/N` declares — a pair that is not two integers, that gives a number no object can have or an offset no member can start at, whose offset is not past the one before it, or whose member would start at or past the end of the decoded data, unless a guard cut it. The header is read as far as it goes, whatever its `/N` claims: the objects listed before the fault are read, the others cannot be read from it. Reported where the stream's data starts, the stream and its fault in the message. An object stream whose `/Filter`, `/DecodeParms` or a value of a step's parameters names an object a guard cut, or left out of the index, is left encoded and not reported: its objects are the guard's, unknown until it is lifted | diff --git a/src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs b/src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs index fa237771..0f53d5ff 100644 --- a/src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs +++ b/src/AdCodicem.Pdf/Diagnostics/PdfDiagnosticCodes.cs @@ -62,14 +62,16 @@ public static class PdfDiagnosticCodes /// /Length gives no length the reader can take and the endstream ends the data, or no endstream /// follows the declared length — none before the next object or the end of the file, or none looked for once the /// document's searches read as much of the file as they may — and that length is kept, or the data runs to the next - /// object. A /Length that leads to an object one of the reader's limits cut, or left out of the index, is not + /// object, or, taken without a search, to the end of the file. A /Length that leads to an object one of the reader's limits cut, or left out of the index, is not /// judged: the limit's report says why. /// public const string StreamLengthInvalid = "stream.length-invalid"; /// - /// A stream has no endstream: its object's endobj ends its data — after its declared length, or where the - /// object ends —, or its data runs to the end of the file. + /// A stream has no endstream before the next object or the end of the file: its object's endobj ends its + /// data — after its declared length, or where the object ends —, or its data runs to the end of the file. Data taken + /// without a search, the document's searches having read as much of the file as they may, is + /// 's, whose endstream no search said is missing. /// public const string StreamTruncated = "stream.truncated"; From 640d9ffd8269d36ad22750ebe424744f8a12271e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 12:24:12 +0000 Subject: [PATCH 049/146] test: reach three paths of a stream's end no test took Patch coverage left three branches untaken that inputs reach: - a parser no file backs, whose buffer ends on the stream keyword or its carriage return, which asks for a larger one; - a deferred /Length whose chain, read whole, gives a length no endstream confirms, which leaves the rows past the endstream they spell to the file; - a stream left unsearched and parsed again after the cache let it go, which says nothing more. Refs #174, #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 7 +++-- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 16 +++++++++++ tests/AdCodicem.Pdf.Tests/ParserTests.cs | 14 ++++++++++ .../AdCodicem.Pdf.Tests/StreamLengthTests.cs | 28 +++++++++++++++++++ 4 files changed, 62 insertions(+), 3 deletions(-) diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 8ae09bb8..f02ee300 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -438,9 +438,10 @@ public static byte[] UpdateNamingItsPrevious(string prev, int padding = 0) /// object 4, which holds 9 0 R, object 9 holding the length so. Its rows of objects 8 and 9 — 10 and 11 when /// —, of reserved types, spell endstream after an end-of-line, and the page's row /// comes after them. The second is a classic table that indexes object 5 — 9 when — again, - /// where it is, and names stream 7 through /Prev. The file is valid. + /// where it is, and names stream 7 through /Prev. The file is valid, unless is added + /// to the length; spaces come before it. /// - public static byte[] UpdatePlacingAPaddedLength(bool chained) + public static byte[] UpdatePlacingAPaddedLength(bool chained, int padding = 3000, int lengthOff = 0) { using var file = new Writer("1.5"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); @@ -448,7 +449,7 @@ public static byte[] UpdatePlacingAPaddedLength(bool chained) file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); var lengthNumber = chained ? 9 : 5; var spelled = "\nendstream\0\0\0\0"u8.ToArray(); - file.Object(lengthNumber, new string(' ', 3000) + (6 * 7 + spelled.Length).ToString(CultureInfo.InvariantCulture)); + file.Object(lengthNumber, new string(' ', padding) + (6 * 7 + spelled.Length + lengthOff).ToString(CultureInfo.InvariantCulture)); if (chained) { diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 698215f7..7f9f3c88 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -522,6 +522,22 @@ public void Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_de new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().BeFalse(); } + [Fact] + public void Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length() + { + // As above, object 9 read whole, with no guard, and 1,000 more than the data's length: nothing confirms it, and the + // rows past the endstream the rows spell are the file's to lack — the section incomplete, the index rebuilt (#220). + var file = ChainFiles.UpdatePlacingAPaddedLength(chained: true, padding: 0, lengthOff: 1000); + using var document = PdfDocument.Open(file); + + document.GetObject(new PdfObjectId(3)); + + var section = document.Reader.Structure.Sections[1]; + section.Incomplete.Should().BeTrue(); + section.CutByLimit.Should().BeFalse(); + document.Diagnostics.Should().NotContain(entry => entry.Code.StartsWith("limit.", StringComparison.Ordinal)); + } + [Theory] [InlineData(31)] [InlineData(32)] diff --git a/tests/AdCodicem.Pdf.Tests/ParserTests.cs b/tests/AdCodicem.Pdf.Tests/ParserTests.cs index 091c3e93..d8cb4680 100644 --- a/tests/AdCodicem.Pdf.Tests/ParserTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ParserTests.cs @@ -202,6 +202,20 @@ public void Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_an value.AsStream().Required().RawLength.Should().Be(length < 0 ? data.IndexOf("6 0 obj", StringComparison.Ordinal) : length); } + [Theory] + [InlineData("4 0 obj\n<< /Length 3 >>\nstream")] + [InlineData("4 0 obj\n<< /Length 3 >>\nstream\r")] + public void Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one(string text) + { + // Parsed alone, which no file backs, a buffer that ends on the stream keyword, or on its carriage return, cannot say + // where the data starts: a larger one says. + var parser = new PdfObjectParser(Encoding.ASCII.GetBytes(text), endsData: false); + + parser.TryReadIndirectObject(out _, out _); + + parser.IsTruncated.Should().BeTrue(); + } + [Fact] public void Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one() { diff --git a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs index 6c0da9aa..0e42a302 100644 --- a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs +++ b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs @@ -2222,6 +2222,34 @@ public void A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_d Invariant($"The stream declared {Declared} bytes, and no endstream follows its data in the bytes read with its object; the {last.Taken} bytes to the end of the file are taken as its data without a search, the document's searches for endstream having read as much of the file as they may.")); } + [Fact] + public void A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed() + { + // The last stream is left unsearched, as above; seventy objects read after it let the cache drop it, and parsed again, + // it keeps its data and says nothing more: its fault was reported the first time. + const int Streams = 30; + var after = Enumerable.Range(Streams + 3, 70).Select(number => Invariant($"{number} 0 obj\n({number})\nendobj\n")).ToArray(); + var (file, _) = NestedStreams(Streams, 10_000_000, after); + using var document = PdfDocument.Open(file, PdfReaderOptions.Default with { ObjectCacheCapacity = 64 }); + + for (var number = 3; number <= Streams + 2; number++) + { + document.GetObject(new PdfObjectId(number)); + } + + document.Reader.TryGetStreamLengthFault(Streams + 2, out var last).Should().BeTrue(); + last.EndStream.Should().Be(EndStreamState.NotSearched); + var length = document.GetObject(new PdfObjectId(Streams + 2)).AsStream().Required().RawLength; + + for (var number = Streams + 3; number < Streams + 73; number++) + { + document.GetObject(new PdfObjectId(number)); + } + + document.GetObject(new PdfObjectId(Streams + 2)).AsStream().Required().RawLength.Should().Be(length); + document.Diagnostics.Should().ContainSingle(d => d.Position == last.DataStart); + } + [Fact] public void A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading() { From 49276043ae4c817c02629218fa496284f5a70adc Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 12:35:36 +0000 Subject: [PATCH 050/146] fix: read an object a window ends exactly with, and blame no endobj on a header the edge cut Three faults of the second review's fixes at a window's edge: - an object whose endobj ended exactly at MaxObjectLength's bound asked for a larger window, which none follows: the guard was reached for an object it holds whole, and the object counted as cut. The file now says whether a regular character follows such an endobj; - a header the edge left undecided ended the containers, and the token after the value, its number, was taken for what follows the object: the endobj was judged missing at the guard's cut. What follows such a header is now unseen; - a number with a sign, or a comment between the tokens, was left undecided as a header, though no header can have either: the value was dropped, and the file's reports of keys that are no names silenced. The bytes read must now keep to the header's rule. Refs #174, #230 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../IO/IPdfStreamDataProvider.cs | 7 ++ src/AdCodicem.Pdf/IO/PdfFileReader.cs | 8 +++ src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 35 +++++++++- tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs | 70 +++++++++++++++++++ 4 files changed, 117 insertions(+), 3 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs index bd1dcfe7..524e349c 100644 --- a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs +++ b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs @@ -31,6 +31,13 @@ internal interface IPdfStreamDataProvider /// long DataStart(long absoluteKeywordEnd); + /// + /// Determines whether a token that reaches the end of the parser's buffer at ends there: + /// the file ends there, or no regular character follows. The parser asks of an endobj its window's edge reaches, + /// which may be the start of a longer keyword (#174). + /// + bool TokenEndsAt(long absoluteOffset); + /// /// Determines whether an endstream keyword follows , after white space and /// comments, however many (#174), and what follows it. The parser asks at the end of a stream's declared length when its diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 6cd54c31..73f05289 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -1551,6 +1551,14 @@ PdfStreamData IPdfStreamDataProvider.Create(long absoluteOffset, int length) return new FileStreamData(this, absoluteOffset, Math.Clamp(length, 0, available), _guard); } + /// + bool IPdfStreamDataProvider.TokenEndsAt(long absoluteOffset) + { + Span next = stackalloc byte[1]; + return absoluteOffset < 0 || absoluteOffset >= _source.Length || _source.Read(absoluteOffset, next) == 0 || + !PdfCharacters.IsRegular(next[0]); + } + /// long IPdfStreamDataProvider.DataStart(long absoluteKeywordEnd) { diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 0e721531..411f42fe 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -295,12 +295,16 @@ public bool TryReadIndirectObject(out PdfObjectId id, out PdfObject value) var afterValue = _lexer.Position; var next = _lexer.Read(); - var unseen = next.Kind == PdfTokenKind.EndOfInput || next.End >= _memory.Length; + + // A header a window's edge left undecided ended the value: what follows it is the edge's, not seen (#230). + var unseen = next.Kind == PdfTokenKind.EndOfInput || next.End >= _memory.Length || _headerUndecided; // What follows a stream whose data runs past the buffer lies past it too: the file was asked at the stream's // end, and its answer stands where the buffer has none (#55). An endobj a window's edge reaches may be the start of - // a longer keyword, which the file read on after a stream says, or a larger window (#174). - var endObj = IsEndObj(next); + // a longer keyword: the file says whether a regular character follows it, so that an object a window ends exactly + // with reads whole, as under a larger window; a parser no file backs asks for a larger buffer (#174). + var endObj = IsEndObj(next) || + (next.IsKeyword("endobj"u8) && !PastWindowEdge && _streamData is { } file && file.TokenEndsAt(_baseOffset + next.End)); _endObj = endObj ? EndObjState.Present : _endObjUnknown ? EndObjState.Unknown @@ -1490,6 +1494,7 @@ private bool StartsHeader(PdfToken number) /// private readonly bool HeaderUndecided(PdfToken number, PdfToken generation, PdfToken keyword) => !_endsData && _member is null && MayBeginHeader(number, generation, _memory.Length) && + MayBeHeaderBytes(number, generation, keyword) && (generation.Kind == PdfTokenKind.EndOfInput || (generation.Kind == PdfTokenKind.Integer && (keyword.Kind == PdfTokenKind.EndOfInput || @@ -1518,6 +1523,30 @@ private readonly bool IsHeader(PdfToken number, PdfToken generation, PdfToken ke !span[generation.End..keyword.Start].ContainsAnyExcept(PdfCharacters.Whitespace); } + /// + /// Determines whether the bytes read of what may be a header, up to the window's edge, keep to the header's byte rule + /// (): digits alone, white space alone between the tokens — no sign, no comment. + /// + private readonly bool MayBeHeaderBytes(PdfToken number, PdfToken generation, PdfToken keyword) + { + var span = _memory.Span; + + if (span[number.Start..number.End].ContainsAnyExceptInRange((byte)'0', (byte)'9')) + { + return false; + } + + if (generation.Kind != PdfTokenKind.Integer) + { + return !span[number.End..].ContainsAnyExcept(PdfCharacters.Whitespace); + } + + var upTo = keyword.Kind == PdfTokenKind.EndOfInput ? span.Length : keyword.Start; + return !span[generation.Start..generation.End].ContainsAnyExceptInRange((byte)'0', (byte)'9') && + !span[number.End..generation.Start].ContainsAnyExcept(PdfCharacters.Whitespace) && + !span[generation.End..upTo].ContainsAnyExcept(PdfCharacters.Whitespace); + } + /// Gets what ended the buffer, as a report of what it cut short names it. private readonly string End => _member switch { diff --git a/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs b/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs index 7793483b..5050a85d 100644 --- a/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs +++ b/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs @@ -319,6 +319,8 @@ public void A_header_the_guard_s_cut_leaves_undecided_gives_the_key_before_it_no [Theory] [InlineData(" /A 1\n\n0 0 obj", 2)] [InlineData(" /A 1 /B 6 70000 obj", 1)] + [InlineData(" /A 1\n\n+6 0 obj", 2)] + [InlineData(" /A 1\n\n6 %c\n0 obj", 2)] public void Tokens_no_header_begins_with_are_reported_at_the_guard_s_cut_as_past_it(string tail, int keys) { // An object number 0, or a generation past 65,535, rules a header out whatever the cut hides: the keys that are not @@ -333,6 +335,74 @@ public void Tokens_no_header_begins_with_are_reported_at_the_guard_s_cut_as_past whole.Diagnostics.Where(entry => entry.Code == PdfDiagnosticCodes.SyntaxUnexpectedToken).Select(entry => entry.Position).Should().Contain(reported); } + [Theory] + [InlineData(" /A 1 /B +6 0 obj")] + [InlineData(" /A 1 /B 6 %c\n0 obj")] + public void A_value_no_header_can_begin_with_keeps_its_place_at_the_guard_s_cut(string tail) + { + // A sign, or a comment between the tokens, rules a header out whatever the cut hides: "+6" is the value of /B, as a + // larger limit reads it, not the start of a header the edge left undecided (#230). + using var document = CutAt(tail, 128); + + document.GetObject(new PdfObjectId(4)).AsDictionary().Required().GetInteger(PdfName.Get("B")).Should().Be(6); + } + + [Theory] + [InlineData("<< /Pad ({0}) /A 1 /B 6 0 objx >>", "obj")] + [InlineData("<< /Pad ({0}) /A 1 /B 6 0 objx >>", "ob")] + [InlineData("<< /Pad ({0}) /A 1 /B 6 0 objx >>", "o")] + [InlineData("<< /Pad ({0}) /A 1 /B [3 6 0 objx] >>", "obj")] + [InlineData("<< /Pad ({0}) /A 1 /B << /D 6 0 objx >> >>", "ob")] + public void An_object_a_guard_cuts_at_a_header_it_leaves_undecided_is_not_said_to_lack_its_endobj(string body, string kept) + { + // Object 4 is whole, its endobj after it; MaxObjectLength ends the window inside "objx", which may be the next object's + // header: the guard's cut, which says nothing of what follows the object (#230). + var text = string.Format(CultureInfo.InvariantCulture, body, new string('x', 40)); + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R /Extra 4 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + [4] = file.Object(4, text), + }; + file.Table(5, rows, "/Size 5 /Root 1 0 R"); + var limit = "4 0 obj\n".Length + text.IndexOf("objx", StringComparison.Ordinal) + kept.Length; + using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = limit } }); + + document.GetObject(new PdfObjectId(4)); + + document.Diagnostics.Should().Contain(entry => entry.Code == PdfDiagnosticCodes.LimitObject); + document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); + new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.ObjectEndObjMissing).Should().BeFalse(); + } + + [Theory] + [InlineData("(abc)")] + [InlineData("<< /A 1 >>")] + [InlineData("[1 2 3]")] + public void An_object_exactly_as_long_as_MaxObjectLength_reads_whole(string body) + { + // The window MaxObjectLength allows ends right after the object's endobj: the file says no regular character goes on + // past it, and the object is whole, as under a larger limit (#174). + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R /Extra 4 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + [4] = file.Object(4, body), + }; + file.Table(5, rows, "/Size 5 /Root 1 0 R"); + var limit = ("4 0 obj\n" + body + "\nendobj").Length; + using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = limit } }); + + document.GetObject(new PdfObjectId(4)); + + // The catalog, longer than the bound, is the guard's; object 4 is not. + document.Diagnostics.Should().NotContain(entry => entry.Position == rows[4]); + document.Reader.IsCutAtLimit(4).Should().BeFalse(); + document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); + } + /// /// Opens a document whose object 4 lost its >> and its endobj, its padding setting 's /// end where a MaxObjectLength of 128 cuts it, under . From dff5738f05f1db61a1ea5158d89f1273e9ea3596 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 12:53:03 +0000 Subject: [PATCH 051/146] fix: read no /Prev of a hybrid file's stream, which no reader follows A hybrid file's cross-reference stream may give a /Prev of its own; the chain goes on through the table's, and readers ignore the stream's. It was read all the same: one leading to an object MaxObjectLength cut ended the chain for the guard, the older table lost, and one no section places was said to name a missing section. The stream's /Prev is no longer read. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 5 ++- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 31 +++++++++++++++++++ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 21 +++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 73f05289..5471375c 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3845,8 +3845,11 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect _rowsLeftUnread = RowsFrom(index is null ? [0, declared] : subsections, stopped, rowsCut ? long.MaxValue : held); _xref.MergeTrailer(dictionary); + + // A hybrid file's stream gives a /Prev of its own, which no reader follows: the chain goes on through the table's, + // and the stream's is not read, to name a section missing or end the chain for a guard. string? previousHidden = null; - previous = rowsCut ? -1 : SectionOffset(dictionary, PdfName.Prev, absolute, out previousHidden); + previous = rowsCut || section.NamedBy == "/XRefStm" ? -1 : SectionOffset(dictionary, PdfName.Prev, absolute, out previousHidden); section.ChainHiddenByGuard = previousHidden; return XRefSectionState.Read; } diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index f02ee300..b9474f9f 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -475,6 +475,37 @@ public static byte[] UpdatePlacingAPaddedLength(bool chained, int padding = 3000 return file.ToArray(); } + /// + /// A hybrid update over a classic table that indexes the catalog, the page tree and the page. The update writes object 9, + /// which holds 123 after spaces, and object 10, which cross-reference stream 7 indexes; the + /// stream's own /Prev is , which no reader follows. The update's table indexes object + /// 9, and its trailer names the older table through /Prev and the stream through /XRefStm. + /// + public static byte[] HybridWithStreamPrev(string streamPrev, int padding = 3000) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + var older = file.Position; + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"xref\n0 4\n0000000000 65535 f\r\n{file.OffsetOf(1):D10} 00000 n\r\n{file.OffsetOf(2):D10} 00000 n\r\n{file.OffsetOf(3):D10} 00000 n\r\ntrailer\n<< /Size 4 /Root 1 0 R >>\nstartxref\n{older}\n%%EOF\n")); + file.Object(9, new string(' ', padding) + "123"); + file.Object(10, "(ten)"); + var stream = file.Position; + var data = Encode([(1, file.OffsetOf(10), 0)], Rows.Raw); + file.Stream( + 7, + string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size 11 /W [1 4 2] /Index [10 1] /Prev {streamPrev} /Length {data.Length}"), + data); + var table = file.Position; + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"xref\n0 1\n0000000000 65535 f\r\n9 1\n{file.OffsetOf(9):D10} 00000 n\r\ntrailer\n<< /Size 11 /Root 1 0 R /Prev {older} /XRefStm {stream} >>\nstartxref\n{table}\n%%EOF\n")); + return file.ToArray(); + } + /// /// A file of two classic tables. The first indexes the catalog, the page tree, the page and object 12, the highest /// number; the second, an update, writes the catalog, the page tree and the page again, indexes them with object 4, a diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 7f9f3c88..0a655066 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -717,6 +717,27 @@ public void A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s(string ke throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); } + [Theory] + [InlineData("9 0 R", true)] + [InlineData("9 0 R", false)] + [InlineData("12 0 R", false)] + public void A_hybrid_stream_s_own_prev_which_no_reader_follows_is_not_read(string prev, bool guarded) + { + // A hybrid file's stream gives a /Prev of its own, which the chain does not follow: it goes on through the table's. + // Read anyway, one that leads to an object MaxObjectLength cuts ended the chain for the guard, the older table lost, + // and one no section places was said to name a missing section (#263). + var file = ChainFiles.HybridWithStreamPrev(prev); + using var document = PdfDocument.Open(file, guarded ? ObjectGuard : null); + + document.WasRepaired.Should().BeFalse(); + document.Reader.Structure.Sections.Should().HaveCount(3); + document.GetObject(new PdfObjectId(3)).AsDictionary().Should().NotBeNull(); + document.Diagnostics.Should().BeEmpty(); + var report = new PdfValidator().Validate(document); + report.Contains(PdfValidationRuleIds.XRefSectionNotFound).Should().BeFalse(); + report.Contains(PdfValidationRuleIds.XRefCheckedInPart).Should().BeFalse(); + } + [Fact] public void A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved() { From 11c4643a0c8f8dbcb5eb21675e5a3dd4f48aad84 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 12:53:03 +0000 Subject: [PATCH 052/146] fix: ask a newer section's cut of a deferred length when the data holds no row whole Rows past an endstream they spell are the guard's when the /Length leads to an object a newer section places and MaxObjectLength cut. That was asked only of data holding a row whole: with the spelled rows first, the data taken to them held none, and the rows past them were charged to the file, which was rebuilt. Data holding no row whole is now asked too. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 5 +++-- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 16 ++++++++++------ tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 14 ++++++++------ 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 5471375c..2acaf661 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3760,8 +3760,9 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // Fewer rows than the section declares, in data a /Length the chain could not read left to the first endstream: // where the section's own row for that /Length, among those held, gives more data, which an endstream confirms, - // the rows spelled an endstream, and the data is taken again to that length (#220). - if (!retaken && deferred is { } unsettled && declared > held && decodedWhole && row > 0) + // the rows spelled an endstream, and the data is taken again to that length (#220). Data that holds no row whole + // is asked too: a newer section may place the object, which a guard cut. + if (!retaken && deferred is { } unsettled && declared > held && decodedWhole) { if (RetakeDeferred(xrefStream, unsettled, data, fieldWidths, row, held, index is null ? [0, declared] : subsections, out lengthCut) is { } whole) { diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index b9474f9f..136c7706 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -439,9 +439,11 @@ public static byte[] UpdateNamingItsPrevious(string prev, int padding = 0) /// —, of reserved types, spell endstream after an end-of-line, and the page's row /// comes after them. The second is a classic table that indexes object 5 — 9 when — again, /// where it is, and names stream 7 through /Prev. The file is valid, unless is added - /// to the length; spaces come before it. + /// to the length; spaces come before it. With , the rows that + /// spell endstream come first: chained, stream 7's row of object 4 then lies past the data taken to that + /// endstream, and the file is read whole only by rebuilding its index. /// - public static byte[] UpdatePlacingAPaddedLength(bool chained, int padding = 3000, int lengthOff = 0) + public static byte[] UpdatePlacingAPaddedLength(bool chained, int padding = 3000, int lengthOff = 0, bool spelledFirst = false) { using var file = new Writer("1.5"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); @@ -457,12 +459,14 @@ public static byte[] UpdatePlacingAPaddedLength(bool chained, int padding = 3000 } var stream = file.Position; - var data = new List(Encode( + var rows = Encode( [(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(chained ? 4 : 5), 0), (1, stream, 0)], - Rows.Raw)); - data.AddRange(spelled); + Rows.Raw); + var data = new List(spelledFirst ? spelled : rows); + data.AddRange(spelledFirst ? rows : spelled); data.AddRange(Encode([(1, file.OffsetOf(3), 0)], Rows.Raw)); - var index = chained ? "[0 3 4 1 7 1 10 2 3 1]" : "[0 3 5 1 7 1 8 2 3 1]"; + var (spelledRows, lengthRow) = chained ? ("10 2", "4 1") : ("8 2", "5 1"); + var index = spelledFirst ? $"[{spelledRows} 0 3 {lengthRow} 7 1 3 1]" : $"[0 3 {lengthRow} 7 1 {spelledRows} 3 1]"; file.Stream( 7, string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size {(chained ? 12 : 10)} /W [1 4 2] /Index {index} /Root 1 0 R /Length {(chained ? 4 : 5)} 0 R"), diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 0a655066..25aca040 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -659,15 +659,17 @@ public void A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chai } [Theory] - [InlineData(false)] - [InlineData(true)] - public void Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut(bool chained) + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + public void Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut( + bool chained, bool spelledFirst) { // Stream 7's /Length leads to an object the newer table places — object 5 itself, or object 9 through object 4, // which only stream 7's own row places — and MaxObjectLength cuts before its value: the rows past the endstream they - // spell are the guard's, as when the cut object is the one the section's own row places (#224). Read whole, the - // file is clean. - var file = ChainFiles.UpdatePlacingAPaddedLength(chained); + // spell are the guard's, as when the cut object is the one the section's own row places (#224) — the data taken to it + // holding no row whole too, the spelled rows first. Read whole, the file is clean. + var file = ChainFiles.UpdatePlacingAPaddedLength(chained, spelledFirst: spelledFirst); using (var whole = PdfDocument.Open(file)) { From c58bcc212257ea4365d7b31f9ae559670466f697 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 12:53:03 +0000 Subject: [PATCH 053/146] fix: check no deferred length against data a decoding guard cut A cross-reference stream's data, taken up to the endstream its rows spell, is taken again to its /Length when the section's own row places it. A guard on decoding the data, such as MaxDecodedStreamLength, keeps it from being taken again; the length, read once the chain is, was then checked against the data and reported as the file's fault, which the whole read does not hold. Only data MaxStreamLength cut was left unchecked; data a decoding guard cut now is. Refs #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 2 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 6 +-- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 37 ++++++++++++------- .../DocumentReaderTests.cs | 21 ++++++++--- 4 files changed, 42 insertions(+), 24 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index 0a20890c..9ac034a0 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -283,7 +283,7 @@ Known gaps: | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s` | Yes | | What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places, the integer read before the cut is only a place to look for `endstream` at: the data is taken again where one confirms it, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, and nothing is charged to the file ([#224]). A row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data `MaxStreamLength` cut is the guard's, and not checked against it ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places, the integer read before the cut is only a place to look for `endstream` at: the data is taken again where one confirms it, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, and nothing is charged to the file ([#224]). A row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, which keeps it from being taken again — is the guard's, and not checked against it ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 2acaf661..91c92cb9 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3830,9 +3830,9 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect } } - // Data a guard cut is the guard's, which says so (ADR 34), and is not checked against a length: decoding it, which - // reads it, says whether it was. - if (deferred is { } check && !xrefStream.Data.CutByGuard) + // Data a guard cut — MaxStreamLength as the data was read, or a guard on decoding it — is the guard's, which says so + // (ADR 34), and is not checked against a length: the cut kept the data from being taken again to it. + if (deferred is { } check && decodedWhole) { (_deferredLengths ??= [])[xrefStream.Data.Position] = check; } diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 136c7706..d7b59f14 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -191,23 +191,31 @@ byte[] Data(long lengthAt, long streamAt) /// A file of one cross-reference stream, object 5, whose /Length names object 4, which only that stream indexes, /// in a row before two rows of reserved types that spell endstream after an end-of-line; the page's row comes /// after them. With , the /Index numbers object 4 twice: first in a row of a - /// reserved type, which the specification says to ignore, then in the row that places it. The file is valid. + /// reserved type, which the specification says to ignore, then in the row that places it. With , + /// the data is compressed in stored blocks, which show the rows as written. The file is valid. /// - public static byte[] AloneWithIndirectLengthBeforeSpelledRows(bool reservedRowFirst) + public static byte[] AloneWithIndirectLengthBeforeSpelledRows(bool reservedRowFirst, bool stored = false) { using var file = new Writer("1.5"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); - var rowCount = reservedRowFirst ? 9 : 8; - file.Object(4, (rowCount * 7).ToString(CultureInfo.InvariantCulture)); - var stream = file.Position; var reserved = reservedRowFirst ? [(3, 0L, 0)] : Array.Empty<(int, long, int)>(); - var data = new List(Encode([(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), .. reserved, (1, file.OffsetOf(4), 0), (1, stream, 0)], Rows.Raw)); - data.AddRange("\nendstream\0\0\0\0"u8.ToArray()); - data.AddRange(Encode([(1, file.OffsetOf(3), 0)], Rows.Raw)); + + // The data's length does not depend on the offsets its rows hold. + byte[] Data(long four, long stream) + { + var data = new List(Encode([(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), .. reserved, (1, four, 0), (1, stream, 0)], Rows.Raw)); + data.AddRange("\nendstream\0\0\0\0"u8.ToArray()); + data.AddRange(Encode([(1, file.OffsetOf(3), 0)], Rows.Raw)); + return stored ? Compress([.. data]) : [.. data]; + } + + file.Object(4, Data(0, 0).Length.ToString(CultureInfo.InvariantCulture)); + var stream = file.Position; var index = reservedRowFirst ? "0 3 4 1 4 2 6 2 3 1" : "0 3 4 2 6 2 3 1"; - file.Stream(5, $"/Type /XRef /Size 8 /W [1 4 2] /Index [{index}] /Root 1 0 R /Length 4 0 R", [.. data]); + var filter = stored ? " /Filter /FlateDecode" : string.Empty; + file.Stream(5, $"/Type /XRef /Size 8 /W [1 4 2] /Index [{index}]{filter} /Root 1 0 R /Length 4 0 R", Data(file.OffsetOf(4), stream)); file.StartXRef(stream); return file.ToArray(); } @@ -561,16 +569,17 @@ private static byte[] Encode((int Type, long Second, int Third)[] rows, Rows enc data.AddRange([(byte)type, (byte)(second >> 24), (byte)(second >> 16), (byte)(second >> 8), (byte)second, (byte)(third >> 8), (byte)third]); } - if (encoding == Rows.Raw) - { - return [.. data]; - } + return encoding == Rows.Raw ? [.. data] : Compress([.. data]); + } + /// Compresses data in stored blocks, which show it as written. + private static byte[] Compress(byte[] data) + { using var output = new MemoryStream(); using (var zlib = new ZLibStream(output, CompressionLevel.NoCompression, leaveOpen: true)) { - zlib.Write([.. data]); + zlib.Write(data); } return output.ToArray(); diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 1bd57681..4e6d160f 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -1901,16 +1901,25 @@ public void A_deferred_length_a_newer_section_refused_takes_no_data_again() sections[1].Fault.Should().Be("it holds 1 row where its /Index and /Size declare 5"); } - [Fact] - public void A_deferred_length_is_not_checked_against_data_a_guard_cut() + [Theory] + [InlineData(false)] + [InlineData(true)] + public void A_deferred_length_is_not_checked_against_data_a_guard_cut(bool decoding) { // The stream's data, taken up to the endstream its rows spell, runs past MaxStreamLength: decoding cut it, and the - // length, read once the chain is, was checked against it, the guard's cut reported as the file's (ADR 34). - var file = ChainFiles.AloneWithIndirectLengthBeforeSpelledRows(reservedRowFirst: false); - using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxStreamLength = 30 } }); + // length, read once the chain is, was checked against it, the guard's cut reported as the file's (ADR 34). Or the + // data, compressed in stored blocks that show the rows as written, decodes past MaxDecodedStreamLength, which kept + // it from being taken again to its length. + var file = ChainFiles.AloneWithIndirectLengthBeforeSpelledRows(reservedRowFirst: false, stored: decoding); + var limits = decoding + ? PdfReaderLimits.Default with { MaxDecodedStreamLength = 30 } + : PdfReaderLimits.Default with { MaxStreamLength = 30 }; + using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = limits }); using var whole = PdfDocument.Open(file); - document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitStreamLength); + document.Diagnostics.Select(entry => entry.Code).Should().Equal( + decoding ? PdfDiagnosticCodes.LimitDecodedStream : PdfDiagnosticCodes.LimitStreamLength); + document.Reader.Structure.Sections.Should().ContainSingle().Which.CutByLimit.Should().BeTrue(); whole.Diagnostics.Should().BeEmpty(); } From 02352570ad721e7963e57e2e9f2a38b6d8efa825 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 12:54:37 +0000 Subject: [PATCH 054/146] docs: say a reference's chain loads a 32nd object, whose value it drops PdfReference.Target's comment said a chain's 31st object was the furthest it is followed to: Resolve loads a 32nd, and drops its value. The 31st is the furthest whose value it takes. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/Objects/PdfReference.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/AdCodicem.Pdf/Objects/PdfReference.cs b/src/AdCodicem.Pdf/Objects/PdfReference.cs index fcfd1e15..767f9b2b 100644 --- a/src/AdCodicem.Pdf/Objects/PdfReference.cs +++ b/src/AdCodicem.Pdf/Objects/PdfReference.cs @@ -62,7 +62,8 @@ internal PdfObjectId Target(int objects = MaxChainLength - 1) { var reference = this; - // Resolve gives the value of a chain's 31st object at most: that object is the furthest a chain is followed to. + // Resolve gives the value of a chain's 31st object at most: it loads a 32nd, whose value it drops (#173), and the 31st + // is the furthest whose value it takes. for (var depth = 1; depth < objects; depth++) { if (reference.Source?.GetObject(reference.Id) is not PdfReference next) From 120779c4ccb80d3d4886ecb0dd1d6e7bdb178036 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 14:09:09 +0000 Subject: [PATCH 055/146] fix: read an object the chain reads exactly within MaxObjectLength whole An object whose endobj ends exactly at MaxObjectLength's bound is read whole by a load, the file saying no regular character follows it. The chain's reads of a /Prev, an /XRefStm, a cross-reference stream's /W or its /Length had no way to ask: such an object ended the chain for the guard, a revision lost, and no load then reached the guard to report it. The parser now asks the file through a small interface of its own, which the chain's reads pass without serving stream data. Refs #174, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/IPdfFileEdge.cs | 20 ++++++++++ .../IO/IPdfStreamDataProvider.cs | 9 +---- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 5 ++- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 11 +++++- tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs | 39 +++++++++++++++++++ 5 files changed, 72 insertions(+), 12 deletions(-) create mode 100644 src/AdCodicem.Pdf/IO/IPdfFileEdge.cs diff --git a/src/AdCodicem.Pdf/IO/IPdfFileEdge.cs b/src/AdCodicem.Pdf/IO/IPdfFileEdge.cs new file mode 100644 index 00000000..87feba7a --- /dev/null +++ b/src/AdCodicem.Pdf/IO/IPdfFileEdge.cs @@ -0,0 +1,20 @@ +namespace AdCodicem.Pdf.IO; + +/// +/// Says what the file holds right past the end of a parser's buffer, where a window's edge leaves a token undecided. +/// +/// +/// A parser given no larger window — one ends — asks it of a +/// keyword its window's edge reaches, which may be the start of a longer one: an object a window ends exactly with reads +/// whole, as under a larger window (#174). It is all the parser asks of the file while the cross-reference chain is read, +/// which reads no stream's data. +/// +internal interface IPdfFileEdge +{ + /// + /// Determines whether a token that reaches the end of the parser's buffer at ends there: + /// the file ends there, or no regular character follows. The parser asks of an endobj its window's edge reaches, + /// which may be the start of a longer keyword (#174). + /// + bool TokenEndsAt(long absoluteOffset); +} diff --git a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs index 524e349c..4ed575ea 100644 --- a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs +++ b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs @@ -11,7 +11,7 @@ namespace AdCodicem.Pdf.IO; /// provider is also the parser's way to the rest of the file, for what its buffer does not hold: the bytes /// after a stream's data, and the objects a stream's /Length names. /// -internal interface IPdfStreamDataProvider +internal interface IPdfStreamDataProvider : IPdfFileEdge { /// Gets the length of the file the stream data is read from. long SourceLength { get; } @@ -31,13 +31,6 @@ internal interface IPdfStreamDataProvider /// long DataStart(long absoluteKeywordEnd); - /// - /// Determines whether a token that reaches the end of the parser's buffer at ends there: - /// the file ends there, or no regular character follows. The parser asks of an endobj its window's edge reaches, - /// which may be the start of a longer keyword (#174). - /// - bool TokenEndsAt(long absoluteOffset); - /// /// Determines whether an endstream keyword follows , after white space and /// comments, however many (#174), and what follows it. The parser asks at the end of a stream's declared length when its diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 91c92cb9..02090b72 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -1552,7 +1552,7 @@ PdfStreamData IPdfStreamDataProvider.Create(long absoluteOffset, int length) } /// - bool IPdfStreamDataProvider.TokenEndsAt(long absoluteOffset) + bool IPdfFileEdge.TokenEndsAt(long absoluteOffset) { Span next = stackalloc byte[1]; return absoluteOffset < 0 || absoluteOffset >= _source.Length || _source.Read(absoluteOffset, next) == 0 || @@ -4862,7 +4862,8 @@ private bool TryReadUnrecorded(PdfObjectId id, long offset, out PdfObject value, { using var window = _source.GetWindow(offset, windowSize); var cut = window.Length == windowSize && offset + window.Length < _source.Length; - var parser = new PdfObjectParser(window.Memory, offset, this, _pending, endsData: !cut); + // The file says where a token the window's edge reaches ends, as it does to a load; no stream's data is read. + var parser = new PdfObjectParser(window.Memory, offset, this, _pending, endsData: !cut, fileEdge: this); var read = parser.TryReadIndirectObject(out var found, out var parsed); // A header read whole that names another object, or another generation, is not this object, however far diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 411f42fe..eedb8b24 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -43,6 +43,7 @@ internal ref struct PdfObjectParser private readonly IPdfObjectSource? _source; private readonly PdfDiagnostics? _diagnostics; private readonly IPdfStreamDataProvider? _streamData; + private readonly IPdfFileEdge? _fileEdge; private readonly long _baseOffset; /// @@ -126,13 +127,18 @@ internal ref struct PdfObjectParser /// Whether the bytes end where the file does; when they end at a window's edge instead, what that edge cuts short is /// not reported, and asks for a larger window. /// + /// + /// Says what the file holds right past the bytes given, for a parser no backs; one that + /// serves stream data says it too. + /// public PdfObjectParser( ReadOnlyMemory memory, long baseOffset = 0, IPdfObjectSource? source = null, PdfDiagnostics? diagnostics = null, IPdfStreamDataProvider? streamData = null, - bool endsData = true) + bool endsData = true, + IPdfFileEdge? fileEdge = null) { _memory = memory; _lexer = new PdfLexer(memory.Span); @@ -140,6 +146,7 @@ public PdfObjectParser( _source = source; _diagnostics = diagnostics; _streamData = streamData; + _fileEdge = fileEdge ?? streamData; _endsData = endsData; } @@ -304,7 +311,7 @@ public bool TryReadIndirectObject(out PdfObjectId id, out PdfObject value) // a longer keyword: the file says whether a regular character follows it, so that an object a window ends exactly // with reads whole, as under a larger window; a parser no file backs asks for a larger buffer (#174). var endObj = IsEndObj(next) || - (next.IsKeyword("endobj"u8) && !PastWindowEdge && _streamData is { } file && file.TokenEndsAt(_baseOffset + next.End)); + (next.IsKeyword("endobj"u8) && !PastWindowEdge && _fileEdge is { } file && file.TokenEndsAt(_baseOffset + next.End)); _endObj = endObj ? EndObjState.Present : _endObjUnknown ? EndObjState.Unknown diff --git a/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs b/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs index 5050a85d..dc8818ed 100644 --- a/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs +++ b/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs @@ -403,6 +403,45 @@ public void An_object_exactly_as_long_as_MaxObjectLength_reads_whole(string body document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); } + [Theory] + [InlineData("Prev", 9)] + [InlineData("XRefStm", 6)] + [InlineData("W", 5)] + [InlineData("Length", 5)] + public void An_object_the_chain_reads_exactly_as_long_as_MaxObjectLength_reads_whole(string key, int number) + { + // The chain reads object N — a /Prev, an /XRefStm, a cross-reference stream's /W or its /Length — through a window + // MaxObjectLength ends right after its endobj: the file says no regular character goes on past it, and the object is + // whole, as a load reads it. It ended the chain for the guard, which no load then reached (#174, #263). + var file = key switch + { + "Prev" => ChainFiles.UpdateNamingItsPrevious("9 0 R", padding: 200), + "XRefStm" => ChainFiles.HybridNamingItsStream("6 0 R", padding: 200), + "W" => ChainFiles.UnderAnUpdate("W", placed: true, padding: 200), + _ => ChainFiles.UpdatePlacingAPaddedLength(chained: false, padding: 200), + }; + var text = Encoding.Latin1.GetString(file); + var header = text.IndexOf(string.Create(CultureInfo.InvariantCulture, $"\n{number} 0 obj\n"), StringComparison.Ordinal) + 1; + var limit = text.IndexOf("endobj", header, StringComparison.Ordinal) + "endobj".Length - header; + using var whole = PdfDocument.Open(file); + + using (var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = limit } })) + { + document.GetObject(new PdfObjectId(3)).AsDictionary().Should().NotBeNull(); + document.Reader.Structure.Sections.Should().OnlyContain(section => !section.CutByLimit && section.ChainHiddenByGuard == null); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(whole.Diagnostics.Select(entry => entry.Code)); + } + + using (var cut = PdfDocument.Open(file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = limit - 1 } })) + { + cut.Diagnostics.Select(entry => entry.Code).Should().Contain(PdfDiagnosticCodes.LimitObject); + } + + var throwing = () => PdfDocument.Open( + file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = limit }, ThrowOnLimit = true }).Dispose(); + throwing.Should().NotThrow(); + } + /// /// Opens a document whose object 4 lost its >> and its endobj, its padding setting 's /// end where a MaxObjectLength of 128 cuts it, under . From d6663f32d3d731c05d0a231c6041573130fb2971 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 14:15:52 +0000 Subject: [PATCH 056/146] fix: search a stream a guard hid the length of past a header its data spells A stream whose /Length leads to an object a guard cut or left out of the index is searched for its endstream, and the search stopped at the first object header its data spells. A valid stream's data may spell one, as an embedded PDF file's does: the data was cut there, and the file said to lack the endstream it holds, a guard's cut charged to the file. Such a stream is now searched on past the header to the next object the index places, and an endstream found there ends its data, as the length would have. With none, the header stops it as before, and the missing endstream is the file's, as settled. The maintainer chose this on 2026-10-23. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/architecture.md | 5 +- docs/threat-model.md | 2 +- docs/website/docs/concepts/diagnostics.md | 5 +- docs/website/docs/reference/diagnostics.md | 4 +- .../IO/IPdfStreamDataProvider.cs | 7 ++- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 57 ++++++++++++++----- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 16 ++++-- src/AdCodicem.Pdf/IO/StreamEnds.cs | 16 +++--- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 42 ++++++++++++++ 9 files changed, 125 insertions(+), 29 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 83337fc6..64b0d6fd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -111,7 +111,9 @@ separate types, separate severity scales, and vocabularies that never share a co each stream — each copy of an object at its offset — takes the same length whatever was read before it. A crafted file can still make it depend on the number a stream is first read under, on the rebuild's scan for trailers, or, once the searches of a document have read four times the file, on which streams were searched - first ([#138], M23). With no `endstream` before the stop, the object's `endobj` ends the data where it + first ([#138], M23). A stream whose `/Length` a guard cut or left out of the index may be valid, its length only + hidden: a header its data spells — an embedded file's does — stops its search only when no `endstream` follows + it before the next object the index places ([#224]). With no `endstream` before the stop, the object's `endobj` ends the data where it follows the declared length as an `endstream` would, or where it is the last one, a whole keyword — no regular character, solidus or percent sign before it, which would make it part of a token, a name or a comment —, and only white space and comments run from it to the stop. That last `endobj` is followed forward through the search's @@ -266,3 +268,4 @@ publication. [#138]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/138 [#170]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/170 [#174]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/174 +[#224]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/224 diff --git a/docs/threat-model.md b/docs/threat-model.md index 9ac034a0..4b07ebfe 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -353,7 +353,7 @@ Known gaps: | A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none` | Property | -| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more` | Plausibly | +| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does: the search goes on past that header up to the next object the index places, and an `endstream` found there ends the data; with none, the header stops it ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.A_stream_a_guard_hid_the_length_of_is_searched_past_a_header_its_data_spells` | Plausibly | | Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, and by white space after a declared length that runs past a first read, about twice the file at most; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | diff --git a/docs/website/docs/concepts/diagnostics.md b/docs/website/docs/concepts/diagnostics.md index 2433bbc8..51a66cd3 100644 --- a/docs/website/docs/concepts/diagnostics.md +++ b/docs/website/docs/concepts/diagnostics.md @@ -67,7 +67,10 @@ A `/Length` that names an object one of the reader's limits cut is no length the null, or `1` of a `15` it parted — is the guard's. An integer read before the cut is only a place to look for `endstream` at, and the data is otherwise found by its `endstream`, as for a stream with no length; nothing is said of the length, while a missing `endstream` is still the file's, and reported. One that names an object a guard left out of -the index is not judged either: its data is found the same way, and a missing `endstream` reported as the file's. +the index is not judged either: its data is found the same way, and a missing `endstream` reported as the file's. Such a +stream may be valid, its length only hidden, and its data may spell an object header, as an embedded PDF file's does: +the search goes on past that header to the next object the index places, and an `endstream` found there ends the data, +as the length would have. With none, the header stops it as any other stream's. A cross-reference stream whose `/Length` the chain cannot read yet, which a valid file may hold, is searched as well, up to the next object the sections read so far place, or the end of the file, and its length read once the chain is. Its diff --git a/docs/website/docs/reference/diagnostics.md b/docs/website/docs/reference/diagnostics.md index c886048a..71bd966a 100644 --- a/docs/website/docs/reference/diagnostics.md +++ b/docs/website/docs/reference/diagnostics.md @@ -184,7 +184,9 @@ said of the length. A missing `endstream` is still the file's, and is reported, cut. A `/Length` that names an object a guard left out of the index, directly or through a chain of references, is not judged either: the data is found by its `endstream` the same way, and a missing one is the file's, the message naming the object the guard left out — `The stream's /Length names object 21 0, which one of the reader's limits left out of the -index, and no endstream follows its data`, then where the search stopped and what was taken. +index, and no endstream follows its data`, then where the search stopped and what was taken. For either, an object +header the data spells, as an embedded PDF file's does, stops the search only when no `endstream` follows it before the +next object the index places: one that does ends the data, and nothing is reported of the stream. ```text Warning stream.length-invalid at 5501: The stream declared 19954 bytes but ended after 19952. diff --git a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs index 4ed575ea..0b95ecbc 100644 --- a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs +++ b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs @@ -67,7 +67,12 @@ internal interface IPdfStreamDataProvider : IPdfFileEdge /// Whether the stream is a cross-reference stream whose /Length the chain could not read (#182): its rows may /// spell an object header and end in a carriage return, so neither stops nor trims the search (#220). /// - StreamEndSearch FindEndStream(int number, long absoluteDataStart, long declaredEnd, ReadOnlySpan buffered, bool deferred); + /// + /// Whether the stream's /Length leads to an object a guard cut or left out of the index, which hides where the + /// data of a stream that may be valid ends: a header its data spells stops the search only when no endstream + /// follows it before the next object the index places, or the end of the file (#224). + /// + StreamEndSearch FindEndStream(int number, long absoluteDataStart, long declaredEnd, ReadOnlySpan buffered, bool deferred, bool lengthHidden); /// /// Resolves the object a stream's /Length names, and says whether the file holds it and whether it could be diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 02090b72..958a6942 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -374,7 +374,7 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// its /Length was one the chain could not read: the file is searched once for each stream, however many times /// it is parsed. With it, the declared end it last said what follows of, read again only when that end changes. /// - private readonly Dictionary<(long DataStart, bool Deferred), (StreamEndSearch Search, long DeclaredEnd)> _endStreamSearches = []; + private readonly Dictionary<(long DataStart, bool Deferred, bool LengthHidden), (StreamEndSearch Search, long DeclaredEnd)> _endStreamSearches = []; /// /// What follows each place the reader read on past white space and comments to find an endstream (#174): read @@ -1780,23 +1780,26 @@ private bool IsEndObjKeywordAt(long position) /// between them, counted over the final index of each of its 401 documents —, and only an invalid stream is searched, /// but for a cross-reference stream whose /Length the chain could not read, which is searched with neither /// stop the rule for headers makes nor a carriage return trimmed, as : its rows may spell - /// a header, and end in a carriage return. What it reads is the stretch between the stream and the next object, + /// a header, and end in a carriage return; and for a stream whose /Length a guard cut or left out of the index, + /// , whose data may spell a header, as an embedded file's does: the search goes on past + /// it to the next object the index places, and the header stops it only when no endstream follows it there + /// (#224). What it reads is the stretch between the stream and the next object, /// once for each stream: the result is kept, and a stream parsed again — after the cache let it go, or a rebuild of /// the index — is not searched again. What the searches read together is bounded by /// . /// /// StreamEndSearch IPdfStreamDataProvider.FindEndStream( - int number, long absoluteDataStart, long declaredEnd, ReadOnlySpan buffered, bool deferred) + int number, long absoluteDataStart, long declaredEnd, ReadOnlySpan buffered, bool deferred, bool lengthHidden) { - var found = _endStreamSearches.TryGetValue((absoluteDataStart, deferred), out var known); + var found = _endStreamSearches.TryGetValue((absoluteDataStart, deferred, lengthHidden), out var known); if (found && known.DeclaredEnd == declaredEnd) { return known.Search; } - var search = found ? known.Search : SearchEndStreamOnce(number, absoluteDataStart, buffered, deferred); + var search = found ? known.Search : SearchEndStreamOnce(number, absoluteDataStart, buffered, deferred, lengthHidden); // Whether the object's endobj follows the declared length depends on the length the stream is read with now, which a // rebuild can change — the /Length an object another definition replaces —: it is read again when that length does, @@ -1810,7 +1813,7 @@ StreamEndSearch IPdfStreamDataProvider.FindEndStream( }; } - _endStreamSearches[(absoluteDataStart, deferred)] = (search, declaredEnd); + _endStreamSearches[(absoluteDataStart, deferred, lengthHidden)] = (search, declaredEnd); return search; } @@ -1819,7 +1822,8 @@ StreamEndSearch IPdfStreamDataProvider.FindEndStream( /// endstream, and reads what its object's endobj does when none is found; what follows the declared length /// is left to . /// - private StreamEndSearch SearchEndStreamOnce(int number, long absoluteDataStart, ReadOnlySpan buffered, bool deferred) + private StreamEndSearch SearchEndStreamOnce( + int number, long absoluteDataStart, ReadOnlySpan buffered, bool deferred, bool lengthHidden) { var nextObject = NextObjectStart(number, absoluteDataStart); @@ -1830,7 +1834,8 @@ private StreamEndSearch SearchEndStreamOnce(int number, long absoluteDataStart, absoluteDataStart, buffered, nextObject ?? _source.Length, - deferred, + stopsAtHeader: !deferred, + keepsCarriageReturn: deferred, readsFile, out var header, out var searched, @@ -1840,6 +1845,30 @@ private StreamEndSearch SearchEndStreamOnce(int number, long absoluteDataStart, // A header the bytes hold is looked for only before the next object the index places: found, it is the nearer. var stop = header >= 0 ? header : nextObject; + // A /Length a guard cut or left out of the index hides where the data of a stream that may be valid ends: data that + // spells an object header, as an embedded file's does, is searched on past it, up to the next object the index + // places, and an endstream found there ends it as the length would have. With none, the header stops it (#224). + if (dataEnd < 0 && header >= 0 && lengthHidden && _endStreamSearchBytes < EndStreamSearchPasses * _source.Length) + { + var pastHeader = SearchEndStream( + header, + [], + nextObject ?? _source.Length, + stopsAtHeader: false, + keepsCarriageReturn: false, + readsFile: true, + out _, + out _, + out _, + out _); + + if (pastHeader >= 0) + { + dataEnd = pastHeader; + stop = nextObject; + } + } + // Past int.MaxValue bytes, no length the reader holds a stream by reaches the endstream found: only a file of more // than 2 GB can put one there, and the stream is read as one whose endstream is not found. With none, what the // object's endobj does is read once with the search, as long as the searches may read. @@ -1975,7 +2004,8 @@ internal static int EndStreamSearchFirstRead(int buffered) => /// Where the data starts in the file. /// The bytes of the data the parser holds, from its start. /// Where the next object the index places starts, or the end of the file. - /// Whether the stream's /Length is one the chain could not read, which no header stops. + /// Whether an object header the bytes hold stops the search. + /// Whether a carriage return before the end-of-line's line feed is data. /// Whether the file past the bytes the parser holds may be read. /// Where the header that stopped the search starts, or -1. /// Whether the search reached where it stops, rather than the end of what it might read. @@ -1988,7 +2018,8 @@ private long SearchEndStream( long dataStart, ReadOnlySpan buffered, long limit, - bool deferred, + bool stopsAtHeader, + bool keepsCarriageReturn, bool readsFile, out long header, out bool searched, @@ -2004,7 +2035,7 @@ private long SearchEndStream( var inBuffer = (int)Math.Clamp(limit - dataStart, 0, buffered.Length); text.Read(buffered[..inBuffer], dataStart); - var found = StreamEnds.SearchStretch(text, endsRange: dataStart + inBuffer >= limit, deferred, out header); + var found = StreamEnds.SearchStretch(text, endsRange: dataStart + inBuffer >= limit, stopsAtHeader, keepsCarriageReturn, out header); endObjSeen = StreamEnds.HoldsEndObj(buffered[..inBuffer], header >= 0 ? (int)(header - dataStart) : inBuffer); tail.Read(buffered[..inBuffer], dataStart, header >= 0 ? header : dataStart + inBuffer); @@ -2045,7 +2076,7 @@ private long SearchEndStream( } text.Read(span, position); - found = StreamEnds.SearchStretch(text, endsRange: position + span.Length >= end, deferred, out header); + found = StreamEnds.SearchStretch(text, endsRange: position + span.Length >= end, stopsAtHeader, keepsCarriageReturn, out header); endObjSeen |= StreamEnds.HoldsEndObj(span, header >= 0 ? (int)(header - position) : span.Length); tail.Read(span, position, header >= 0 ? header : position + span.Length); @@ -3618,7 +3649,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // file, ends nowhere the section says: its rows are not read (#220). Where the /Length leads to an object a guard cut, // the length that would have said is the guard's, as a value its rows are read with is (#263): the section counts as // cut, the chain ends here, and the guard is reached once the chain is read. - if (deferred is not null && _endStreamSearches.TryGetValue((xrefStream.Data.Position, true), out var known) && + if (deferred is not null && _endStreamSearches.TryGetValue((xrefStream.Data.Position, true, false), out var known) && known.Search is { Length: null } search) { // The fault names the link of the chain that could not be read — the one no section read so far places, or the diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index eedb8b24..61e59c27 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -730,7 +730,8 @@ private PdfObject ParseDictionaryOrStream(int depth, int openedAt) /// The declared length is confirmed by an endstream after it, white space and comments between, however many. /// Unconfirmed, or with no length to confirm, the data is searched for its first endstream, which ends it: in the /// bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold and the next - /// object the index places, or to the end of the data. Stream data never counts toward the bytes the parser is given + /// object the index places, or to the end of the data — past such a header when a guard hid the length, as long as an + /// endstream follows it before that object. Stream data never counts toward the bytes the parser is given /// to read an object through: a window is never grown to find an endstream. With none before the stop, the /// object's endobj plays its part: it ends the data where it follows the declared length, or where it ends the /// object before the stop; without it, a declared length that ends before the stop is kept, and the data otherwise @@ -793,11 +794,18 @@ private PdfStream ReadStream(PdfDictionary dictionary) // A cross-reference stream whose /Length the chain could not read keeps a carriage return before the line feed as // data, which it may be — rows can end in 0x0D, and the length, once read, is checked against what was taken —, - // and is not stopped by rows that spell an object header (#182, #220). + // and is not stopped by rows that spell an object header (#182, #220). One whose /Length a guard cut or left out of + // the index is searched on past a header its data spells, which a valid stream's may (#224). var deferred = declared.Form == StreamLengthForm.Deferred; var declaredEnd = length < 0 ? -1 : _baseOffset + dataStart + length; var search = _streamData is not null - ? _streamData.FindEndStream(_objectNumber, _baseOffset + dataStart, declaredEnd, span[Math.Min(dataStart, span.Length)..], deferred) + ? _streamData.FindEndStream( + _objectNumber, + _baseOffset + dataStart, + declaredEnd, + span[Math.Min(dataStart, span.Length)..], + deferred, + lengthHidden: declared.Form is StreamLengthForm.ObjectCut or StreamLengthForm.ObjectUnindexed) : SearchBuffer(span, dataStart, declaredEnd, deferred); if (search is not { } settled) @@ -996,7 +1004,7 @@ static string Where(long? nextObject) => nextObject is { } at var text = new HeaderText(carried, carriedAt, start); text.Read(span[dataStart..], start); - var dataEnd = StreamEnds.SearchStretch(text, endsRange: _endsData, deferred, out var header); + var dataEnd = StreamEnds.SearchStretch(text, endsRange: _endsData, stopsAtHeader: !deferred, keepsCarriageReturn: deferred, out var header); if (dataEnd >= 0) { diff --git a/src/AdCodicem.Pdf/IO/StreamEnds.cs b/src/AdCodicem.Pdf/IO/StreamEnds.cs index 258f82d4..51e8bd9c 100644 --- a/src/AdCodicem.Pdf/IO/StreamEnds.cs +++ b/src/AdCodicem.Pdf/IO/StreamEnds.cs @@ -114,19 +114,21 @@ public static bool HoldsEndObj(ReadOnlySpan bytes, int before) => /// /// /// The overlap between reads puts the end-of-line before a keyword first seen in a read inside it, unless the keyword - /// starts the data. A cross-reference stream whose /Length the chain could not read is searched as - /// : its rows may spell a header, and may end in a carriage return, so neither stops nor - /// trims it (#220). + /// starts the data. A cross-reference stream whose /Length the chain could not read is searched neither stopped + /// by a header nor trimmed of a carriage return: its rows may spell a header, and may end in a carriage return (#220). + /// A stream whose /Length a guard hid is searched on past a header its data spells, once the header stopped it + /// (#224). /// /// The bytes read, after what the reads before carry into them. /// Whether they end where the search does: at the next object the index places, or the end of the file. - /// Whether the stream's /Length is one the chain could not read, which is no fault. + /// Whether an object header the bytes hold before the keyword stops the search. + /// Whether a carriage return before the end-of-line's line feed is data. /// Where the header that stopped the search starts in the file, or -1. - public static long SearchStretch(in HeaderText text, bool endsRange, bool deferred, out long header) + public static long SearchStretch(in HeaderText text, bool endsRange, bool stopsAtHeader, bool keepsCarriageReturn, out long header) { var span = text.Bytes; var keyword = span.IndexOf(EndStreamKeyword); - header = deferred ? -1 : text.FindHeader(keyword >= 0 ? keyword : span.Length, endsRange); + header = stopsAtHeader ? text.FindHeader(keyword >= 0 ? keyword : span.Length, endsRange) : -1; if (keyword < 0 || header >= 0) { @@ -138,7 +140,7 @@ public static long SearchStretch(in HeaderText text, bool endsRange, bool deferr keyword--; } - if (!deferred && keyword > 0 && span[keyword - 1] == (byte)'\r') + if (!keepsCarriageReturn && keyword > 0 && span[keyword - 1] == (byte)'\r') { keyword--; } diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 25aca040..10bcbc4c 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -183,6 +183,48 @@ public void A_missing_endstream_is_the_file_s_fault_when_a_guard_left_the_length fault.Reference.Should().Be(new PdfObjectId(21)); } + [Theory] + [InlineData("cut", true)] + [InlineData("left out", true)] + [InlineData("cut", false)] + [InlineData("left out", false)] + public void A_stream_a_guard_hid_the_length_of_is_searched_past_a_header_its_data_spells(string guard, bool endStream) + { + // Object 4 embeds a PDF file, whose data spells "1 0 obj", and its /Length, object 10, is one MaxObjectLength cuts or + // MaxIndexEntries leaves out of the index: searched for its endstream, the data stopped at the header it spells, and + // the file was said to lack the endstream it holds (#224). Without one, the header still stops it, the file's fault. + const string Embedded = "%PDF-1.4\n1 0 obj\n<< /Type /Catalog >>\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF"; + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + [4] = file.Object(4, "<< /Type /EmbeddedFile /Length 10 0 R >>\nstream\n" + Embedded + (endStream ? "\nendstream" : string.Empty)), + [6] = file.Object(6, "(six)"), + [10] = file.Object(10, (guard == "cut" ? new string(' ', 2000) : string.Empty) + Embedded.Length.ToString(CultureInfo.InvariantCulture)), + }; + file.Table(11, rows, "/Size 11 /Root 1 0 R"); + var limits = guard == "cut" + ? PdfReaderLimits.Default with { MaxObjectLength = 1024 } + : PdfReaderLimits.Default with { MaxIndexEntries = 8 }; + using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = limits }); + + var data = document.GetObject(new PdfObjectId(4)).AsStream().Required().GetRawBytes().ToArray(); + var codes = document.Diagnostics.Select(entry => entry.Code).ToArray(); + + if (endStream) + { + data.Should().Equal(Encoding.ASCII.GetBytes(Embedded)); + codes.Should().Equal(guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries); + document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); + } + else + { + data.Should().Equal("%PDF-1.4\n"u8.ToArray()); + codes.Should().Contain(PdfDiagnosticCodes.StreamLengthInvalid); + } + } + [Theory] [InlineData(false)] [InlineData(true)] From 9e98575c44a60b44329a437a0a65dd3af7463a4a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 14:18:43 +0000 Subject: [PATCH 057/146] fix: take a deferred length's data again to what a guard's cut left wherever it lies Rows that spell an endstream are told from the data's end by the integer the cut left of the /Length object the section's own row places, when MaxObjectLength cut it. Where the chain ends at an object a guard cut through a reference that object holds, or the /Length names an object a newer section places, the rows past the spelled endstream went to the guard without that integer being tried, though an endstream confirmed it. It is now read where the sections read so far place the cut object, and the data taken again to it when an endstream confirms it. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 2 +- .../docs/reference/validation-rules.md | 2 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 67 ++++++++++++++----- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 9 ++- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 22 ++++++ 5 files changed, 80 insertions(+), 22 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index 4b07ebfe..f11b6ac7 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -283,7 +283,7 @@ Known gaps: | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s` | Yes | | What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places, the integer read before the cut is only a place to look for `endstream` at: the data is taken again where one confirms it, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, and nothing is charged to the file ([#224]). A row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, which keeps it from being taken again — is the guard's, and not checked against it ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at: the data is taken again where one confirms it, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, which keeps it from being taken again — is the guard's, and not checked against it ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | diff --git a/docs/website/docs/reference/validation-rules.md b/docs/website/docs/reference/validation-rules.md index e661ea76..c287eae9 100644 --- a/docs/website/docs/reference/validation-rules.md +++ b/docs/website/docs/reference/validation-rules.md @@ -51,7 +51,7 @@ References are to ISO 32000-1:2008. | Rule | Severity | Meaning | Reads | |---|---|---|---| -| `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]); or a cross-reference stream whose `/Length` is such a reference, or leads through references to an object no section read before it places, and whose data no `endstream` ends before the next object the sections read so far place, or the end of the file — or was not searched for one, the document's searches for `endstream` having read as much of the file as they may ([#220]). Rows that spell `endstream` end such a stream's data there, and the section lacks the rows past them, unless the section's own row for that `/Length` — the row the reading of its rows keeps, one of a reserved type ignored —, among the rows before them, places an object that gives a longer length an `endstream` confirms: the data is then taken to it. A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. Nor is a cross-reference stream whose value the rows are read with leads to an object a limit cut: the section counts as cut there, the chain ends at it, nothing is rebuilt, and `xref.checked-in-part` says so ([#263]). Nor is one whose `/Length`, which the chain could not read, leads to an object `MaxObjectLength` cuts, where only that length could say where the data ends: with no `endstream` before the next object or the end of the file, the section counts as cut, as above; when its rows spell one, the integer the cut left of the object the section's own row places is where an `endstream` is looked for, the data taken to it when one confirms it, and otherwise the rows past the spelled one the limit's, the section counting as cut there, as above ([#224]). | Each section of the chain | +| `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]); or a cross-reference stream whose `/Length` is such a reference, or leads through references to an object no section read before it places, and whose data no `endstream` ends before the next object the sections read so far place, or the end of the file — or was not searched for one, the document's searches for `endstream` having read as much of the file as they may ([#220]). Rows that spell `endstream` end such a stream's data there, and the section lacks the rows past them, unless the section's own row for that `/Length` — the row the reading of its rows keeps, one of a reserved type ignored —, among the rows before them, places an object that gives a longer length an `endstream` confirms: the data is then taken to it. A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. Nor is a cross-reference stream whose value the rows are read with leads to an object a limit cut: the section counts as cut there, the chain ends at it, nothing is rebuilt, and `xref.checked-in-part` says so ([#263]). Nor is one whose `/Length`, which the chain could not read, leads to an object `MaxObjectLength` cuts, where only that length could say where the data ends: with no `endstream` before the next object or the end of the file, the section counts as cut, as above; when its rows spell one, the integer the cut left of that object — whether the section's own row places it, or a newer section does — is where an `endstream` is looked for, the data taken to it when one confirms it, and otherwise the rows past the spelled one the limit's, the section counting as cut there, as above ([#224]). | Each section of the chain | | `xref.section-not-found` | Error | A section a `/Prev` or an `/XRefStm` names is neither there nor within 512 bytes of it, or what names it is not an offset — tiff2pdf writes `/Prev 576066 0 R`. The objects only it indexes are missing from the index. The section `startxref` names is `file.startxref-wrong`'s. A `/Prev` or an `/XRefStm` written as a reference that leads to an object `MaxObjectLength` cuts names no section missing: the chain ends there for the limit, the sections older than it unread, which `xref.checked-in-part` says ([#263]). | Each section of the chain | | `xref.section-shifted` | Warning | A section a `/Prev` or an `/XRefStm` names starts a few bytes from where it is named, and the reader found it there. | Each section of the chain | | `xref.chain-loop` | Error | The chain names a section it has already read, or an offset it has already named where it read none, in the file or outside it: the reader stops there, and the section the chain should have gone on to is unknown, what only it indexed found, if at all, by rebuilding the index — as for `xref.section-not-found`. | The chain | diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 958a6942..1ab4d240 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3898,11 +3898,12 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect /// reserved type is ignored there, as the specification says (). The object is read /// where that row places it as reads one: nothing is corrected, cached, recorded or /// reported. One a guard cuts gives what the cut left of it, an integer read before the cut only a place to look for - /// endstream at (#224), and says so when no endstream confirms it — as it does - /// when the object holds a reference that leads to an object a guard cut, or when a newer section gave the number and - /// the object it places is one, or leads to one. A row that does not stand otherwise — a newer section gave the number, - /// or refused it; the row lies past those held, places the object in an object stream, or under another generation — - /// leaves the data as it was taken. + /// endstream at (#224), and says so when no endstream confirms it. So does the + /// object a guard cut at the end of a chain of references from the object read, or from the /Length when a newer + /// section gave the number: the integer the cut left of that object, read where the sections read so far place it, is + /// the place to look at then (). A row that does not stand otherwise — a newer section gave + /// the number, or refused it; the row lies past those held, places the object in an object stream, or under another + /// generation — leaves the data as it was taken. /// private PdfStream? RetakeDeferred( PdfStream stream, @@ -3920,9 +3921,9 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect if (_xref.TryGet(number, out _) || _refusedNumbers?.Contains(number) == true) { // The object a newer section places, which the chain could not read: a guard that cut it, or the object a chain - // of references from it leads to, leaves how far the data runs unknown. - lengthCut = LeadsToCut(stream.Dictionary.GetRaw(PdfName.Length), LengthLinks); - return null; + // of references from it leads to, leaves how far the data runs to the integer the cut left, where an endstream + // confirms it, and unknown otherwise. + return TakeAgainAtCut(stream, deferred, stream.Dictionary.GetRaw(PdfName.Length), LengthLinks, out lengthCut); } // The rows that number the object, in the order the subsections give them, up to the first of a type the reading @@ -3962,21 +3963,53 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect } var read = TryReadUnrecorded(deferred.Length, PositionOf((long)offset), out var value, out var pastGuard); - var dataStart = stream.Data.Position; - if ((read || pastGuard) && value.AsInteger() is { } length && length > deferred.Taken && length <= int.MaxValue && - length <= _source.Length - dataStart && ((IPdfStreamDataProvider)this).CheckEndStream(dataStart + length) is not null) + if ((read || pastGuard) && TakeAgain(stream, deferred, value) is { } whole) { - return new PdfStream(stream.Dictionary, ((IPdfStreamDataProvider)this).Create(dataStart, (int)length)); + return whole; } // The object read holds the chain's second link: the chain goes on as far as the parse resolves a /Length through. - lengthCut = pastGuard || (value is PdfReference && LeadsToCut(value, LengthLinks - 1)); - return null; + lengthCut = pastGuard; + return !pastGuard && value is PdfReference ? TakeAgainAtCut(stream, deferred, value, LengthLinks - 1, out lengthCut) : null; + } + + /// + /// Gives cross-reference stream with its data taken again to the length + /// gives, when it gives more than the data taken and an endstream confirms it; null + /// otherwise. + /// + private PdfStream? TakeAgain(PdfStream stream, DeferredLength deferred, PdfObject value) + { + var dataStart = stream.Data.Position; + + return value.AsInteger() is { } length && length > deferred.Taken && length <= int.MaxValue && + length <= _source.Length - dataStart && ((IPdfStreamDataProvider)this).CheckEndStream(dataStart + length) is not null + ? new PdfStream(stream.Dictionary, ((IPdfStreamDataProvider)this).Create(dataStart, (int)length)) + : null; + } + + /// + /// Gives cross-reference stream with its data taken again, when , read + /// through the sections read so far and references, leads to an object a guard cut: to the + /// integer the cut left of it, read where they place it, when an endstream confirms it (#224). Null otherwise, + /// and says whether a guard cut that object, which leaves how far the data runs unknown. + /// + private PdfStream? TakeAgainAtCut(PdfStream stream, DeferredLength deferred, PdfObject? written, int links, out bool lengthCut) + { + Follow(PdfName.Length, written, entry: true, out _, out var cut, links); + lengthCut = cut is not null; + + // The object the guard cut is one a section read so far places, which ReadWithoutLoading read there. + if (cut is not { } id || !_xref.TryGet(id.Number, out var entry)) + { + return null; + } - // Whether a value read through the sections read so far leads to an object a guard cut. - bool LeadsToCut(PdfObject? written, int links) => - Follow(PdfName.Length, written, entry: true, out _, out var cut, links) is not null && cut is not null; + TryReadUnrecorded(id, PositionOf(entry), out var value, out _); + var whole = TakeAgain(stream, deferred, value); + lengthCut = whole is null; + return whole; } /// diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index d7b59f14..27714289 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -449,9 +449,11 @@ public static byte[] UpdateNamingItsPrevious(string prev, int padding = 0) /// where it is, and names stream 7 through /Prev. The file is valid, unless is added /// to the length; spaces come before it. With , the rows that /// spell endstream come first: chained, stream 7's row of object 4 then lies past the data taken to that - /// endstream, and the file is read whole only by rebuilding its index. + /// endstream, and the file is read whole only by rebuilding its index. With , the + /// spaces come after the length instead. /// - public static byte[] UpdatePlacingAPaddedLength(bool chained, int padding = 3000, int lengthOff = 0, bool spelledFirst = false) + public static byte[] UpdatePlacingAPaddedLength( + bool chained, int padding = 3000, int lengthOff = 0, bool spelledFirst = false, bool lengthFirst = false) { using var file = new Writer("1.5"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); @@ -459,7 +461,8 @@ public static byte[] UpdatePlacingAPaddedLength(bool chained, int padding = 3000 file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); var lengthNumber = chained ? 9 : 5; var spelled = "\nendstream\0\0\0\0"u8.ToArray(); - file.Object(lengthNumber, new string(' ', padding) + (6 * 7 + spelled.Length + lengthOff).ToString(CultureInfo.InvariantCulture)); + var length = (6 * 7 + spelled.Length + lengthOff).ToString(CultureInfo.InvariantCulture); + file.Object(lengthNumber, lengthFirst ? length + new string(' ', padding) : new string(' ', padding) + length); if (chained) { diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 10bcbc4c..50c7491e 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -548,6 +548,28 @@ public void The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_s new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().BeFalse(); } + [Theory] + [InlineData(false, 0)] + [InlineData(true, 0)] + [InlineData(false, 3)] + [InlineData(true, 3)] + public void The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to( + bool chained, int lengthOff) + { + // Stream 7's /Length leads to an object the newer table places — object 5 itself, or object 9 through object 4, + // which only stream 7's own row places — and MaxObjectLength cuts it after its value. The value the cut left is where + // to look for the endstream, as for an object the section's own row places: confirmed, the data is taken again to it + // and every row read; otherwise the rows past the endstream they spell are the guard's (#224). + var file = ChainFiles.UpdatePlacingAPaddedLength(chained, lengthOff: lengthOff, lengthFirst: true); + using var document = PdfDocument.Open(file, ObjectGuard); + + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.Reader.Structure.Sections[1].CutByLimit.Should().Be(lengthOff != 0); + document.Reader.Structure.Sections[1].Incomplete.Should().BeFalse(); + (document.GetObject(new PdfObjectId(3)).AsDictionary() is not null).Should().Be(lengthOff == 0); + } + [Fact] public void Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value() { From 5c50289b69d2c93d12e091a99114a83ac92ecf52 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 14:21:01 +0000 Subject: [PATCH 058/146] fix: say a hidden /XRefStm leaves its revision's stream unchecked, not older sections xref.checked-in-part said a section whose /XRefStm leads to an object MaxObjectLength cut "names older sections ... they were not checked". A hybrid file's /XRefStm names the cross-reference stream of the section's own revision, and the table may have no /Prev at all: the message claimed older sections that do not exist. The section now records which value hid the chain, and the rule says the stream of its revision, and any section older than it, were not checked. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../docs/reference/validation-rules.md | 4 +-- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 1 + .../IO/XRef/XRefSectionRecord.cs | 7 ++++++ .../Validation/Rules/CheckedInPartRule.cs | 12 ++++++--- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 25 +++++++++++++------ 5 files changed, 36 insertions(+), 13 deletions(-) diff --git a/docs/website/docs/reference/validation-rules.md b/docs/website/docs/reference/validation-rules.md index c287eae9..18910abc 100644 --- a/docs/website/docs/reference/validation-rules.md +++ b/docs/website/docs/reference/validation-rules.md @@ -52,7 +52,7 @@ References are to ISO 32000-1:2008. | Rule | Severity | Meaning | Reads | |---|---|---|---| | `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]); or a cross-reference stream whose `/Length` is such a reference, or leads through references to an object no section read before it places, and whose data no `endstream` ends before the next object the sections read so far place, or the end of the file — or was not searched for one, the document's searches for `endstream` having read as much of the file as they may ([#220]). Rows that spell `endstream` end such a stream's data there, and the section lacks the rows past them, unless the section's own row for that `/Length` — the row the reading of its rows keeps, one of a reserved type ignored —, among the rows before them, places an object that gives a longer length an `endstream` confirms: the data is then taken to it. A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. Nor is a cross-reference stream whose value the rows are read with leads to an object a limit cut: the section counts as cut there, the chain ends at it, nothing is rebuilt, and `xref.checked-in-part` says so ([#263]). Nor is one whose `/Length`, which the chain could not read, leads to an object `MaxObjectLength` cuts, where only that length could say where the data ends: with no `endstream` before the next object or the end of the file, the section counts as cut, as above; when its rows spell one, the integer the cut left of that object — whether the section's own row places it, or a newer section does — is where an `endstream` is looked for, the data taken to it when one confirms it, and otherwise the rows past the spelled one the limit's, the section counting as cut there, as above ([#224]). | Each section of the chain | -| `xref.section-not-found` | Error | A section a `/Prev` or an `/XRefStm` names is neither there nor within 512 bytes of it, or what names it is not an offset — tiff2pdf writes `/Prev 576066 0 R`. The objects only it indexes are missing from the index. The section `startxref` names is `file.startxref-wrong`'s. A `/Prev` or an `/XRefStm` written as a reference that leads to an object `MaxObjectLength` cuts names no section missing: the chain ends there for the limit, the sections older than it unread, which `xref.checked-in-part` says ([#263]). | Each section of the chain | +| `xref.section-not-found` | Error | A section a `/Prev` or an `/XRefStm` names is neither there nor within 512 bytes of it, or what names it is not an offset — tiff2pdf writes `/Prev 576066 0 R`. The objects only it indexes are missing from the index. The section `startxref` names is `file.startxref-wrong`'s. A `/Prev` or an `/XRefStm` written as a reference that leads to an object `MaxObjectLength` cuts names no section missing: the chain ends there for the limit, what it names unread, which `xref.checked-in-part` says ([#263]). | Each section of the chain | | `xref.section-shifted` | Warning | A section a `/Prev` or an `/XRefStm` names starts a few bytes from where it is named, and the reader found it there. | Each section of the chain | | `xref.chain-loop` | Error | The chain names a section it has already read, or an offset it has already named where it read none, in the file or outside it: the reader stops there, and the section the chain should have gone on to is unknown, what only it indexed found, if at all, by rebuilding the index — as for `xref.section-not-found`. | The chain | | `xref.entry-broken` | Error | An in-use entry places its object neither at its offset nor within 512 bytes of it, in an object stream whose header does not list it, or in one where it is written as a stream or holds one, which an object stream cannot hold (7.5.7), and which reads as null — the index the reader rebuilt included, where it places such an object and the file's own index places nothing, or the part of it the reader read where one of its limits stopped the chain short ([#161]) — as a load decides it: the header read at the offset after white space and comments, however long, a whole token no header begins with ruling one out at once, and the one found near it confirmed by what a load parses there ([#175], [#256]). An offset outside the file — past its end, before its start, or past what a `long` holds once the header's offset is added — is looked for nearby as a load looks: an object found there is `xref.entry-shifted`'s. One finding per entry, located at the object and the offset the entry gives, or at the object alone when that offset lies outside the file. | Each object's header, as a load reads it, a few dozen bytes in a sound file; each object stream's header, and each member whose bytes hold `stream`, parsed as a load parses it | @@ -63,7 +63,7 @@ References are to ISO 32000-1:2008. | `xref.object-stream-value-wrong` | Warning | An object stream gives its `/N`, `/First` or `/Length` as a real number with no fractional part, where the specification asks for an integer (Tables 5 and 16). The reader reads it as the integer it equals, as MuPDF, PDFBox and pdf.js do; qpdf and poppler lose the stream's objects ([#215]). One finding per stream and key. A value with a fractional part is another fault: a `/N` or a `/First` leaves the stream unreadable, `xref.object-stream-broken`'s, and a `/Length` is recovered through the stream's `endstream`, which the reader reports (`stream.length-invalid`). | As `xref.object-stream-circular` | | `xref.offset-imprecise` | Warning | Offsets name the white space before what they designate rather than its first byte (7.5.4, 7.5.5): Microsoft Print to PDF names the line feed before each object, and some tools the one before `xref`. Every reader skips the white space. One finding for the file, with how many offsets and the first of them. | The sections of the chain, and each object's header | | `xref.object-past-size` | Warning | In-use objects are numbered above the trailer's `/Size`, which Table 15 makes a conforming reader ignore ("shall be ignored and defined to be missing"). The entries, the objects' headers and the references agree against `/Size`, and the reader reads those objects, as qpdf does while warning. One finding for the file, with how many objects and the first of them; `file.size-wrong` then says nothing more of that `/Size`. | The index and the trailer's `/Size` | -| `xref.checked-in-part` | Information | Part of the index was not checked: one of the reader's limits stopped it reading a section, a trailer or an object stream whole, the chain past `MaxXRefSectionCount` sections, or the index past the `MaxIndexEntries` object numbers it holds, said once, where it filled — raising the limit lets the rest be checked —; an entry whose header, where it places its object or near it, runs past `MaxObjectLength`; the entries the probe left undecided once reading their headers had consumed twice the file's length — the bytes crossed up to each decision, past the first 64 of each, and every byte read past a run an entry skips, which a valid file's headers, distinct bytes, never reach —, said once ([#175]); the sections older than one whose `/Prev` or `/XRefStm` leads to an object `MaxObjectLength` cuts, which end the chain there, said at that section with the reference that names them ([#263]); or the document is encrypted and its object streams are readable only once decrypted (M16). A chain cut where it goes on outside the file is located at the section whose `/Prev` names where. | What the reader and the probe of the index recorded | +| `xref.checked-in-part` | Information | Part of the index was not checked: one of the reader's limits stopped it reading a section, a trailer or an object stream whole, the chain past `MaxXRefSectionCount` sections, or the index past the `MaxIndexEntries` object numbers it holds, said once, where it filled — raising the limit lets the rest be checked —; an entry whose header, where it places its object or near it, runs past `MaxObjectLength`; the entries the probe left undecided once reading their headers had consumed twice the file's length — the bytes crossed up to each decision, past the first 64 of each, and every byte read past a run an entry skips, which a valid file's headers, distinct bytes, never reach —, said once ([#175]); the sections older than one whose `/Prev` leads to an object `MaxObjectLength` cuts — or the cross-reference stream of its revision, and any section older than it, when its `/XRefStm` does —, which end the chain there, said at that section with the reference that names them ([#263]); or the document is encrypted and its object streams are readable only once decrypted (M16). A chain cut where it goes on outside the file is located at the section whose `/Prev` names where. | What the reader and the probe of the index recorded | The entries the `xref.entry-*`, `xref.generation-mismatch`, `xref.object-stream-*`, `xref.offset-imprecise` and `xref.object-past-size` rules check are those of the index the file's chain gave, as the reader first read diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 1ab4d240..49572dea 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3115,6 +3115,7 @@ private bool TryReadClassicTable(long absolute, XRefSectionRecord section, out l previous = SectionOffset(trailer, PdfName.Prev, absolute, out var previousHidden); hybrid = SectionOffset(trailer, PdfName.XRefStm, absolute, out var streamHidden); section.ChainHiddenByGuard = streamHidden ?? previousHidden; + section.ChainHiddenAtStream = streamHidden is not null; // A stream a guard hid holds rows newer than the older sections': the chain ends with it, as with a // stream a guard cut, since their rows would stand for the stream's. diff --git a/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs b/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs index 5913ca53..64a8466b 100644 --- a/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs +++ b/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs @@ -80,6 +80,12 @@ public XRefSectionRecord(string namedBy, long namedOffset, long namedFrom) /// public string? ChainHiddenByGuard { get; set; } + /// + /// Gets a value indicating whether what says is the section's /XRefStm: what + /// the guard hides is then the cross-reference stream of the section's revision, and any section older than it. + /// + public bool ChainHiddenAtStream { get; set; } + /// Gets the section's own trailer — a cross-reference stream's dictionary —, or null when none was read. public PdfDictionary? Trailer { get; set; } @@ -126,6 +132,7 @@ public void RelocateTo(XRefSectionRecord attempt) CutByLimit = attempt.CutByLimit; TrailerCutByLimit = attempt.TrailerCutByLimit; ChainHiddenByGuard = attempt.ChainHiddenByGuard; + ChainHiddenAtStream = attempt.ChainHiddenAtStream; Trailer = attempt.Trailer; TrailerFault = attempt.TrailerFault; TrailerPosition = attempt.TrailerPosition; diff --git a/src/AdCodicem.Pdf/Validation/Rules/CheckedInPartRule.cs b/src/AdCodicem.Pdf/Validation/Rules/CheckedInPartRule.cs index e95cbb8d..ccc367a5 100644 --- a/src/AdCodicem.Pdf/Validation/Rules/CheckedInPartRule.cs +++ b/src/AdCodicem.Pdf/Validation/Rules/CheckedInPartRule.cs @@ -72,14 +72,20 @@ public void Check(ValidationContext context) remedy: null); } + // A hybrid file's /XRefStm names the cross-reference stream of the section's own revision, which a /Prev may + // not follow: what the guard hides is that stream, and whatever is older. if (section.ChainHiddenByGuard is { } hidden) { context.Report( this, PdfValidationLocation.AtPosition(section.Offset), - string.Create( - CultureInfo.InvariantCulture, - $"The cross-reference section at offset {section.Offset} names older sections, but {hidden}: they were not checked."), + section.ChainHiddenAtStream + ? string.Create( + CultureInfo.InvariantCulture, + $"The cross-reference section at offset {section.Offset} names the cross-reference stream of its revision, but {hidden}: that stream, and any section older than it, were not checked.") + : string.Create( + CultureInfo.InvariantCulture, + $"The cross-reference section at offset {section.Offset} names older sections, but {hidden}: they were not checked."), remedy: null); } } diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 50c7491e..e0b40276 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -762,7 +762,8 @@ public void A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s(string ke // The newer section's /Prev, object 9, or its /XRefStm, object 6, is one that section places and MaxObjectLength cuts // before its value: the section it names was said to be missing, and the index rebuilt. The chain ends there for the // guard, which is reached once the chain is read — thrown at opening when the caller asks for it (#263). The - // validation says the sections it names were not checked, and judges nothing the chain they end would hold. + // validation says what it names was not checked — the older sections, or the stream of the hybrid's revision —, and + // judges nothing the chain they end would hold. var reference = key == "Prev" ? "9 0 R" : "6 0 R"; var file = key == "Prev" ? ChainFiles.UpdateNamingItsPrevious(reference, padding: 3000) @@ -777,7 +778,9 @@ public void A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s(string ke report.Contains(PdfValidationRuleIds.FileSizeWrong).Should().BeFalse(); var section = document.Reader.Structure.Sections[0].Offset; report.Findings.Where(finding => finding.RuleId == PdfValidationRuleIds.XRefCheckedInPart).Select(finding => finding.Message).Should().Equal( - $"The cross-reference section at offset {section} names older sections, but its /{key} is the reference {reference}, which one of the reader's limits cut: they were not checked."); + key == "Prev" + ? $"The cross-reference section at offset {section} names older sections, but its /Prev is the reference {reference}, which one of the reader's limits cut: they were not checked." + : $"The cross-reference section at offset {section} names the cross-reference stream of its revision, but its /XRefStm is the reference {reference}, which one of the reader's limits cut: that stream, and any section older than it, were not checked."); var throwing = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); @@ -804,12 +807,17 @@ public void A_hybrid_stream_s_own_prev_which_no_reader_follows_is_not_read(strin report.Contains(PdfValidationRuleIds.XRefCheckedInPart).Should().BeFalse(); } - [Fact] - public void A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved() + [Theory] + [InlineData("Prev")] + [InlineData("XRefStm")] + public void A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved(string key) { - // A third table's /Prev names the table whose /Prev, object 9, MaxObjectLength cuts, 3 bytes before it: the section is - // found where it lies, and what its /Prev hides is said of it there, as where it is named (#263). - var two = ChainFiles.UpdateNamingItsPrevious("9 0 R", padding: 3000); + // A third table's /Prev names the table whose /Prev, object 9, or whose /XRefStm, object 6, MaxObjectLength cuts, 3 + // bytes before it: the section is found where it lies, and what that value hides is said of it there, as where it is + // named (#263). + var two = key == "Prev" + ? ChainFiles.UpdateNamingItsPrevious("9 0 R", padding: 3000) + : ChainFiles.HybridNamingItsStream("6 0 R", padding: 3000); var text = Encoding.Latin1.GetString(two); var named = long.Parse(text[(text.LastIndexOf("startxref\n", StringComparison.Ordinal) + "startxref\n".Length)..].Split('\n')[0], CultureInfo.InvariantCulture); var five = two.Length; @@ -825,7 +833,8 @@ public void A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is moved.ChainHiddenByGuard.Should().NotBeNull(); new PdfValidator().Validate(document).Findings .Where(finding => finding.RuleId == PdfValidationRuleIds.XRefCheckedInPart) - .Should().ContainSingle(finding => finding.Message.Contains("names older sections", StringComparison.Ordinal)); + .Should().ContainSingle(finding => finding.Message.Contains( + key == "Prev" ? "names older sections" : "names the cross-reference stream of its revision", StringComparison.Ordinal)); } [Fact] From c7c0de7f0b53cd6d438fe1a33d532875c579c61b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 14:22:54 +0000 Subject: [PATCH 059/146] test: hold how far the chain follows a /Length or /Prev on the paths no test reached Review found four paths whose rule no test held, each mutation of it passing the whole suite: - a cross-reference stream's own /Prev that a guard cut, which ends the chain as a table's does; - a deferred /Length followed 32 objects where the data has no endstream, and where the rows spell one, from the object a newer section places and from what the own row's object holds; - a deferred /Length's chain longer than the parse follows, reported at opening as a load reads it; - the missing endstream's report naming the 32nd object of a chain a guard cut or left out of the index. Each new test fails under the mutation that review named, and passes on the code as it is. Refs #220, #224, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 89 +++++++++++ .../DocumentReaderTests.cs | 32 ++++ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 148 ++++++++++++++++++ 3 files changed, 269 insertions(+) diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 27714289..cec473ba 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -490,6 +490,64 @@ public static byte[] UpdatePlacingAPaddedLength( return file.ToArray(); } + /// + /// A file as writes it, the rows that spell endstream numbering objects 8 + /// and 9, but for a length a chain of objects gives: stream 7's /Length names object 5 + /// — or, with , object 4, which only stream 7's own row places —, which holds 100 0 R; + /// object 100 names object 101, and so on to the last, which holds the data's length after 3,000 spaces. The newer table + /// indexes object 5, unless , and objects 100 to the last, where they are. The file is valid. + /// + public static byte[] UpdatePlacingAPaddedLengthChain(int objects, bool ownRow) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + var lengthNumber = ownRow ? 4 : 5; + var last = 100 + objects - 2; + var spelled = "\nendstream\0\0\0\0"u8.ToArray(); + file.Object(lengthNumber, "100 0 R"); + + for (var number = 100; number <= last; number++) + { + file.Object( + number, + number < last + ? string.Create(CultureInfo.InvariantCulture, $"{number + 1} 0 R") + : new string(' ', 3000) + (6 * 7 + spelled.Length).ToString(CultureInfo.InvariantCulture)); + } + + var stream = file.Position; + var data = new List(Encode( + [(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(lengthNumber), 0), (1, stream, 0)], + Rows.Raw)); + data.AddRange(spelled); + data.AddRange(Encode([(1, file.OffsetOf(3), 0)], Rows.Raw)); + file.Stream( + 7, + string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size 10 /W [1 4 2] /Index [0 3 {lengthNumber} 1 7 1 8 2 3 1] /Root 1 0 R /Length {lengthNumber} 0 R"), + [.. data]); + file.Text("%" + new string('-', 1200) + "\n"); + var xref = file.Position; + var rows = new StringBuilder("xref\n0 1\n0000000000 65535 f\r\n"); + + if (!ownRow) + { + rows.Append(CultureInfo.InvariantCulture, $"5 1\n{file.OffsetOf(5):D10} 00000 n\r\n"); + } + + rows.Append(CultureInfo.InvariantCulture, $"100 {objects - 1}\n"); + + for (var number = 100; number <= last; number++) + { + rows.Append(CultureInfo.InvariantCulture, $"{file.OffsetOf(number):D10} 00000 n\r\n"); + } + + rows.Append(CultureInfo.InvariantCulture, $"trailer\n<< /Size {last + 1} /Root 1 0 R /Prev {stream} >>\nstartxref\n{xref}\n%%EOF\n"); + file.Text(rows.ToString()); + return file.ToArray(); + } + /// /// A hybrid update over a classic table that indexes the catalog, the page tree and the page. The update writes object 9, /// which holds 123 after spaces, and object 10, which cross-reference stream 7 indexes; the @@ -521,6 +579,37 @@ public static byte[] HybridWithStreamPrev(string streamPrev, int padding = 3000) return file.ToArray(); } + /// + /// A classic table updated by a cross-reference stream. The table indexes the catalog, the page tree, the page and + /// object 12, the highest number; cross-reference stream 10, the update, indexes the first three with object 4, a page + /// the tree does not list, and object 9, which holds the table's offset after spaces, and + /// names the table through /Prev 9 0 R. The table and the stream both give /Size 13. + /// + public static byte[] StreamUpdateHidingItsHighestNumber(int padding) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + file.Object(12, "(twelve)"); + var first = file.Position; + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"xref\n0 4\n0000000000 65535 f\r\n{file.OffsetOf(1):D10} 00000 n\r\n{file.OffsetOf(2):D10} 00000 n\r\n{file.OffsetOf(3):D10} 00000 n\r\n12 1\n{file.OffsetOf(12):D10} 00000 n\r\ntrailer\n<< /Size 13 /Root 1 0 R >>\nstartxref\n{first}\n%%EOF\n")); + file.Object(4, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + file.Object(9, new string(' ', padding) + first.ToString(CultureInfo.InvariantCulture)); + var stream = file.Position; + var data = Encode( + [(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), (1, file.OffsetOf(4), 0), (1, file.OffsetOf(9), 0), (1, stream, 0)], + Rows.Raw); + file.Stream( + 10, + string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size 13 /W [1 4 2] /Index [0 5 9 2] /Root 1 0 R /Prev 9 0 R /Length {data.Length}"), + data); + file.StartXRef(stream); + return file.ToArray(); + } + /// /// A file of two classic tables. The first indexes the catalog, the page tree, the page and object 12, the highest /// number; the second, an update, writes the catalog, the page tree and the page again, indexes them with object 4, a diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 4e6d160f..219f7457 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -2144,6 +2144,38 @@ public void A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_t "The /Length of cross-reference stream 7 names object 5 0, which holds the name /Foo, not a length; the chain took its data up to its endstream, after 56 bytes."); } + [Theory] + [InlineData(31, false)] + [InlineData(32, false)] + [InlineData(33, true)] + [InlineData(34, true)] + public void A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it(int objects, bool tooLong) + { + // Stream 7's /Length names object 5, which the newer table places, and which names object 100, the next, and so on; + // the last holds 56, and none is cut. The chain read follows as many objects as the parse resolves a /Length through, + // and reads a longer chain as null, as a later load of stream 7 does: the length is reported at opening, and the load + // adds nothing. Once, the parse resolved the reference the chain read gave back, took 56, and the report waited for + // the load (#220, #224). + var placed = new Dictionary(); + var last = 100 + objects - 2; + + for (var number = 100; number <= last; number++) + { + placed[number] = number < last ? string.Create(CultureInfo.InvariantCulture, $"{number + 1} 0 R") : "56"; + } + + var file = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] /Length 5 0 R", "100 0 R", compressed: false, predicted: false, placed: true, placedToo: placed); + string[] expected = tooLong ? [PdfDiagnosticCodes.StreamLengthInvalid] : []; + using var document = PdfDocument.Open(file); + + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(expected); + + document.GetObject(new PdfObjectId(7)).AsStream().Required().GetRawBytes().Length.Should().Be(56); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(expected); + } + [Fact] public void A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain() { diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index e0b40276..18bb8707 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -639,6 +639,55 @@ public void A_length_whose_chain_ends_at_its_last_object_a_guard_cut_is_the_guar document.Reader.TryGetStreamLengthFault(4, out _).Should().BeFalse(); } + [Theory] + [InlineData(31, "cut")] + [InlineData(32, "cut")] + [InlineData(31, "left out of the index")] + [InlineData(32, "left out of the index")] + public void A_missing_endstream_names_the_last_object_of_a_length_s_chain_a_guard_cut_or_left_out_of_the_index(int objects, string guard) + { + // As above, content stream 4 having lost its endstream, and the last object of its /Length's chain cut after 3,000 + // spaces — or placed only by an older table, which MaxXRefSectionCount leaves unread: the fault and the report of the + // missing endstream name that object, the 32nd as the 31st, and not the one before it, which no guard cut and the + // index holds (#224). + var file = new HandWrittenFile(); + var last = 100 + objects - 1; + var unindexed = guard == "left out of the index"; + var rows = new Dictionary(); + var trailer = Invariant($"/Size {last + 1} /Root 1 0 R"); + + if (unindexed) + { + var older = file.Table(last + 1, new Dictionary { [last] = file.Object(last, "5") }, Invariant($"/Size {last + 1}")); + trailer += Invariant($" /Prev {older}"); + } + else + { + rows[last] = file.Object(last, new string(' ', 3000) + "5"); + } + + rows[1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + rows[2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"); + rows[4] = file.Object(4, "<< /Length 100 0 R >>\nstream\n0 0 m"); + + for (var number = 100; number < last; number++) + { + rows[number] = file.Object(number, Invariant($"{number + 1} 0 R")); + } + + // The newer table numbers no row for the last object when only the older one places it. + file.Table(unindexed ? last : last + 1, rows, trailer); + var options = unindexed ? new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxXRefSectionCount = 1 } } : ObjectGuard; + using var document = PdfDocument.Open(file.Bytes, options); + + document.GetObject(new PdfObjectId(4)).AsStream().Required().GetRawBytes().ToArray().Should().Equal("0 0 m"u8.ToArray()); + document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.StreamTruncated).Which.Message.Should().StartWith( + Invariant($"The stream's /Length names object {last} 0, which one of the reader's limits {guard}, and no endstream follows its data")); + document.Reader.TryGetStreamLengthFault(4, out var fault).Should().BeTrue(); + fault.Form.Should().Be(unindexed ? StreamLengthForm.ObjectUnindexed : StreamLengthForm.ObjectCut); + fault.Reference.Should().Be(new PdfObjectId(last)); + } + [Theory] [InlineData(31)] [InlineData(32)] @@ -673,6 +722,71 @@ public void A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_objec throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); } + [Theory] + [InlineData(31)] + [InlineData(32)] + public void A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s(int objects) + { + // As above, stream 7 having lost its endstream: the fault that says its data ends nowhere follows the /Length as far + // as the parse resolves one, so the 32nd object, cut, leaves the section to the guard — not a chain of references + // that reaches no value, which rebuilt the index (#220, #224). + var placed = new Dictionary(); + var last = 100 + objects - 2; + + for (var number = 100; number <= last; number++) + { + placed[number] = number < last ? Invariant($"{number + 1} 0 R") : new string(' ', 3000) + "56"; + } + + var file = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] /Length 5 0 R", "100 0 R", compressed: false, predicted: false, placed: true, placedToo: placed); + var at = Encoding.Latin1.GetString(file).IndexOf("endstream", StringComparison.Ordinal); + Encoding.ASCII.GetBytes("endstrXam").CopyTo(file, at); + using var document = PdfDocument.Open(file, ObjectGuard); + + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.Reader.Structure.Sections[1].CutByLimit.Should().BeTrue(); + document.Reader.Structure.Sections[1].Fault.Should().EndWith(Invariant($"the reference {last} 0 R, which one of the reader's limits cut")); + + var throwing = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); + throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); + } + + [Theory] + [InlineData(31, false)] + [InlineData(32, false)] + [InlineData(31, true)] + [InlineData(32, true)] + public void Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut(int objects, bool ownRow) + { + // Stream 7's /Length names object 5, which the newer table places — or object 4, which only stream 7's own row + // places —, which names object 100, the next, and so on; the last, after 3,000 spaces, holds the length, and the rows + // spell an endstream. Whether the length leads to a cut follows it as far as the parse resolves one, from the object + // the newer table places or from what the own row's object holds: the 32nd, cut, leaves the rows past the endstream + // to the guard — the page, whose row is one, reads as null —, where a shorter walk gave them to the file, the section + // incomplete and the index rebuilt (#220, #224). + var file = ChainFiles.UpdatePlacingAPaddedLengthChain(objects, ownRow); + + using (var whole = PdfDocument.Open(file)) + { + whole.GetObject(new PdfObjectId(3)).AsDictionary().Should().NotBeNull(); + whole.Diagnostics.Should().BeEmpty(); + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + document.GetObject(new PdfObjectId(3)).Should().BeOfType(); + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + var section = document.Reader.Structure.Sections[1]; + section.CutByLimit.Should().BeTrue(); + section.Incomplete.Should().BeFalse(); + + var throwing = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); + throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); + } + [Theory] [InlineData(31)] [InlineData(32)] @@ -786,6 +900,40 @@ public void A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s(string ke throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); } + [Fact] + public void A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s() + { + // Cross-reference stream 10's /Prev, object 9, is one its own rows place, and MaxObjectLength cuts before its value: + // the chain ends there for the guard, as at a table's /Prev, thrown at opening when the caller asks for it (#263). + // Read whole, the stream's /Size 13 is one more than object 12, which only the older table indexes, and object 4 is + // a page the tree does not list: the validation says the table was not checked, and judges neither on the stream. + var file = ChainFiles.StreamUpdateHidingItsHighestNumber(padding: 3000); + + using (var whole = PdfDocument.Open(file)) + { + whole.Reader.Structure.Sections.Should().HaveCount(2); + var report = new PdfValidator().Validate(whole); + report.Contains(PdfValidationRuleIds.FileSizeWrong).Should().BeFalse(); + report.Contains(PdfValidationRuleIds.PageTreePageOrphaned).Should().BeTrue(); + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + var section = document.Reader.Structure.Sections.Should().ContainSingle().Which; + section.Kind.Should().Be(XRefSectionKind.Stream); + section.ChainHiddenByGuard.Should().NotBeNull(); + var guarded = new PdfValidator().Validate(document); + guarded.Findings.Where(finding => finding.RuleId == PdfValidationRuleIds.XRefCheckedInPart).Select(finding => finding.Message).Should().Equal( + $"The cross-reference section at offset {section.Offset} names older sections, but its /Prev is the reference 9 0 R, which one of the reader's limits cut: they were not checked."); + guarded.Contains(PdfValidationRuleIds.FileSizeWrong).Should().BeFalse(); + guarded.Contains(PdfValidationRuleIds.PageTreePageOrphaned).Should().BeFalse(); + + var throwing = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); + throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); + } + [Theory] [InlineData("9 0 R", true)] [InlineData("9 0 R", false)] From 82f9f6e6250ed95261db54897d50726995146d33 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 14:24:47 +0000 Subject: [PATCH 060/146] docs: cite the tests that hold the threat model's stream and chain rows The threat model's rows on the header rule, the chain's references, a deferred /Length, a /Length a guard cut and a stream's end cited none of the tests the review rounds added for them; each now does, and the header row says an object exactly as long as MaxObjectLength reads whole. The texts on an endobj that a solidus or a percent sign makes no keyword say that only the byte right before it is read: a comment further back on the line does not hide it. Refs #174, #220, #224, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/architecture.md | 3 ++- docs/threat-model.md | 16 ++++++++-------- src/AdCodicem.Pdf/IO/EndObjTail.cs | 8 +++++--- tests/AdCodicem.Pdf.Tests/ParserTests.cs | 4 ++-- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 64b0d6fd..2629238a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -115,7 +115,8 @@ separate types, separate severity scales, and vocabularies that never share a co hidden: a header its data spells — an embedded file's does — stops its search only when no `endstream` follows it before the next object the index places ([#224]). With no `endstream` before the stop, the object's `endobj` ends the data where it follows the declared length as an `endstream` would, or where it is the last one, a whole keyword — no regular - character, solidus or percent sign before it, which would make it part of a token, a name or a comment —, and only + character, solidus or percent sign right before it, which would make it part of a token, a name or a comment that + starts there: the data before it, which no lexing reads, says nothing of one further back —, and only white space and comments run from it to the stop. That last `endobj` is followed forward through the search's own reads, which read every byte up to the stop already: no byte before the stop is read a second time, backward, and a comment, which can be told only from the start of its line, is read as one. Without either, diff --git a/docs/threat-model.md b/docs/threat-model.md index f11b6ac7..e29bc7a1 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -248,7 +248,7 @@ Known gaps: | The lexer looping, or reading past its buffer | Every token but the end of input consumes a byte. Look-aheads check the length, and a string that never closes ends at the buffer's end, and is told from a closed one by its token's length | `PropertyTests.The_lexer_terminates_on_any_bytes_and_stays_inside_the_buffer`; `LexerTests.Never_loops_on_a_stray_delimiter`, `Recovers_from_an_unterminated_literal_string`; `ParserTests.Reports_nothing_of_a_construct_closed_at_the_end_of_the_data` | Property, over buffers of about 100 bytes | | Nesting that exhausts the stack | `PdfObjectParser.MaxDepth` (128) and a stack probe at each container. Past either, `SkipContainer` counts brackets without recursion, and `syntax.depth-exceeded` is reported | `ParserTests.Refuses_to_follow_nesting_deeper_than_its_limit`; `HostileInputTests.Survives_deeply_nested_containers_inside_an_object`, `A_file_nesting_objects_deeper_than_the_stack_allows_reads_them_as_null_without_crashing` | No | | A token or an object of any length, or one that never closes | The window grows ×8 from 8 KB up to `MaxObjectLength`, and the parser is told whether its window ends the data: only then is what it leaves open reported, once, `syntax.truncated-object` where the innermost construct opens, and a window that ends where the file does cuts nothing however long it is. Past the bound, the object is kept as far as it was read and `limit.object` is reported, beside what the parse met before the cut; nothing is reported of the token the cut reaches, nor of what follows once a look-ahead met it (ADR 34). An array or a dictionary an `endobj` finds open ends there, rather than take the objects after it, and one that lost its own `endobj` too ends at the next object's header (below). The parses of one object add up to about 1.3 times the last window | `HostileInputTests.Survives_a_string_longer_than_the_largest_window`, `A_string_that_never_closes_takes_the_end_of_the_file_and_is_reported_once_where_it_opens`; `ReaderLimitsTests.An_object_is_read_no_further_than_its_bound`, `An_object_cut_by_its_bound_is_not_reported_as_damage`, `An_object_its_bound_cuts_anywhere_in_its_last_tokens_reports_the_bound_alone`, `What_an_object_its_bound_cuts_met_before_the_cut_is_still_reported`, `A_fault_after_a_stream_whose_length_the_file_cannot_hold_is_still_reported_beside_the_bound`, `An_object_its_bound_cuts_after_more_faults_than_the_diagnostics_hold_still_reports_the_bound`, `A_number_its_bound_cuts_is_no_number_out_of_range`, `An_object_as_long_as_its_bound_that_ends_the_file_is_read_whole`; `WindowEdgeTests.An_anomaly_inside_an_object_longer_than_the_window_is_reported_once`; `PropertyTests.What_a_window_reports_of_its_syntax_the_whole_data_reports_too_in_the_same_order`, `A_buffer_that_ends_at_a_window_s_edge_reports_nothing_of_the_end_of_the_data`; `SyntaxFaultTests.A_construct_the_file_ends_inside_is_reported_where_it_opens`, `A_container_an_endobj_ends_takes_nothing_of_the_objects_after_it`, `A_fault_of_an_object_read_through_a_window_the_file_goes_on_past_is_reported`, `A_rebuild_reports_no_fault_where_its_window_cuts_a_trailer`, `A_trailer_a_source_cut_short_ends_inside_is_reported_as_the_end_of_the_data`; `FileRuleTests.A_trailer_whose_closing_brackets_end_its_table_s_window_is_judged_by_what_they_close` | Growth yes, the bound no; the window's edge by property | -| An object that lost both its closing delimiter and its `endobj` taking in the next object, up to that object's `endobj`, its own missing `endobj` unreported; a trailer that lost its `>>` taking the next update's entries; a header the window's edge cut short reported as the file's stray keys | Inside an array or a dictionary, where an element, a key or a value is expected, the next object's header ends every container left open: `N G obj`, digits alone, `N` in 1..`int.MaxValue` and `G` in 0..65,535, white space alone between them, and `obj` a whole keyword the window's edge does not reach — the header the search for a lost `endstream` stops at (`HeaderText`). A header the edge cuts short, `6 0` or `6 0 ob`, is the edge's, as a reference it cuts is: where no larger window follows — at `MaxObjectLength`, in the rebuild's trailer scan —, its numbers are not reported as keys that are not names, unless a whole token already rules a header out — a number 0, a generation past 65,535, a member, which no header follows —; `6 0 ob`, a keyword `obj` starts after two integers, which no reference can be, ends the containers as a header does, its number given to no key and nothing reported, while `6 0` alone, what the edge may have left of a reference, is kept as far as it was read, as any value the edge cuts. It is reported once, at the innermost container, as `syntax.truncated-object`, "An object header ended the object inside …", or, in a trailer or a value read alone, where no object is open, "An object header came inside …", and the object lacks its `endobj`. Containers `SkipContainer` passes over past `MaxDepth` end there too, and a trailer is read so as well. An object stream's member, which no header follows, and a value alone, which is no container, do not apply it: in `4 0 obj 5 0 obj`, object 4 holds the integer 5. Its cost, accepted: a closed container holding text that reads as a header ends there, and its `endobj` is judged missing; none of the 344,361 objects of the corpus holds one ([#230]) | `SyntaxFaultTests.An_object_that_lost_its_closing_delimiter_and_its_endobj_ends_at_the_next_object_s_header`, `The_next_object_s_header_ends_every_container_left_open_and_is_reported_at_the_innermost`, `Text_that_is_no_object_header_ends_no_container`, `A_header_inside_a_closed_container_ends_it_there_and_its_endobj_is_judged_missing`, `An_object_stream_member_holding_a_header_reads_it_as_tokens`, `Nesting_deeper_than_the_reader_follows_ends_at_the_next_object_s_header_too`, `A_trailer_that_lost_its_closing_delimiter_takes_nothing_of_the_update_after_it`, `A_header_the_guard_s_cut_leaves_undecided_is_the_guard_s`; `ParserTests.Ends_the_containers_an_object_left_open_at_the_next_object_s_header_and_leaves_it_to_be_read`, `Takes_an_object_header_a_window_s_edge_reaches_for_none_until_a_larger_window_shows_obj_whole`, `A_header_where_a_key_or_a_value_is_expected_ends_the_dictionary`, `A_value_alone_that_is_a_header_s_number_is_an_integer`, `Tokens_a_window_s_edge_leaves_undecided_as_a_header_are_no_stray_keys`; `WindowEdgeTests.An_object_that_lost_its_end_ends_at_the_next_header_wherever_the_window_edge_cuts_it` | Plausibly | +| An object that lost both its closing delimiter and its `endobj` taking in the next object, up to that object's `endobj`, its own missing `endobj` unreported; a trailer that lost its `>>` taking the next update's entries; a header the window's edge cut short reported as the file's stray keys | Inside an array or a dictionary, where an element, a key or a value is expected, the next object's header ends every container left open: `N G obj`, digits alone, `N` in 1..`int.MaxValue` and `G` in 0..65,535, white space alone between them, and `obj` a whole keyword the window's edge does not reach — the header the search for a lost `endstream` stops at (`HeaderText`). A header the edge cuts short, `6 0` or `6 0 ob`, is the edge's, as a reference it cuts is: where no larger window follows — at `MaxObjectLength`, in the rebuild's trailer scan —, its numbers are not reported as keys that are not names, unless a whole token already rules a header out — a number 0, a generation past 65,535, a member, which no header follows —; `6 0 ob`, a keyword `obj` starts after two integers, which no reference can be, ends the containers as a header does, its number given to no key and nothing reported, while `6 0` alone, what the edge may have left of a reference, is kept as far as it was read, as any value the edge cuts. It is reported once, at the innermost container, as `syntax.truncated-object`, "An object header ended the object inside …", or, in a trailer or a value read alone, where no object is open, "An object header came inside …", and the object lacks its `endobj`. Containers `SkipContainer` passes over past `MaxDepth` end there too, and a trailer is read so as well. An object stream's member, which no header follows, and a value alone, which is no container, do not apply it: in `4 0 obj 5 0 obj`, object 4 holds the integer 5. Its cost, accepted: a closed container holding text that reads as a header ends there, and its `endobj` is judged missing; none of the 344,361 objects of the corpus holds one ([#230]). An `endobj` the window of `MaxObjectLength` ends exactly with is asked of the file, which says whether a regular character goes on past it: an object exactly as long as the guard reads whole, loaded or read by the chain ([#174]) | `SyntaxFaultTests.An_object_that_lost_its_closing_delimiter_and_its_endobj_ends_at_the_next_object_s_header`, `The_next_object_s_header_ends_every_container_left_open_and_is_reported_at_the_innermost`, `Text_that_is_no_object_header_ends_no_container`, `A_header_inside_a_closed_container_ends_it_there_and_its_endobj_is_judged_missing`, `An_object_stream_member_holding_a_header_reads_it_as_tokens`, `Nesting_deeper_than_the_reader_follows_ends_at_the_next_object_s_header_too`, `A_trailer_that_lost_its_closing_delimiter_takes_nothing_of_the_update_after_it`, `A_header_the_guard_s_cut_leaves_undecided_is_the_guard_s`; `ParserTests.Ends_the_containers_an_object_left_open_at_the_next_object_s_header_and_leaves_it_to_be_read`, `Takes_an_object_header_a_window_s_edge_reaches_for_none_until_a_larger_window_shows_obj_whole`, `A_header_where_a_key_or_a_value_is_expected_ends_the_dictionary`, `A_value_alone_that_is_a_header_s_number_is_an_integer`, `Tokens_a_window_s_edge_leaves_undecided_as_a_header_are_no_stray_keys`; `WindowEdgeTests.An_object_that_lost_its_end_ends_at_the_next_header_wherever_the_window_edge_cuts_it`; `SyntaxFaultTests.A_header_the_guard_s_cut_leaves_undecided_gives_the_key_before_it_no_value`, `A_value_no_header_can_begin_with_keeps_its_place_at_the_guard_s_cut`, `An_object_a_guard_cuts_at_a_header_it_leaves_undecided_is_not_said_to_lack_its_endobj`, `Tokens_no_header_begins_with_are_reported_at_the_guard_s_cut_as_past_it`, `An_object_exactly_as_long_as_MaxObjectLength_reads_whole`, `An_object_the_chain_reads_exactly_as_long_as_MaxObjectLength_reads_whole`; `ParserTests.Ends_a_container_skipped_for_its_depth_at_the_next_object_s_header`, `Reports_the_keys_of_a_member_a_window_s_edge_leaves_as_a_header_none_can_be` | Plausibly | | Look-ahead that makes parsing superlinear | Look-ahead is a constant number of tokens, so each token is lexed three times at most | `ParserTests.Distinguishes_a_reference_from_two_integers`, `Backtracks_when_two_integers_are_not_followed_by_R` | Property | | A number past a `long` or a `double` | The number parser never wraps: an integer past a `long` reads as the real nearest to it, and a number past a double's range as an infinity, which the object parser reads as null under `syntax.number-out-of-range`, quoting the number only in a report the diagnostics keep. A real reads as the double nearest to its decimal. The shape — signs, then digits around at most one period — is checked before the framework's parse, which would read `NaN` and `Infinity`; each digit is read twice at most, and nothing is allocated | `LexerTests.Reads_an_integer_too_large_for_a_long_as_a_real`, `Reads_a_number_past_the_largest_double_as_an_infinity_its_reader_can_tell`, `Parsing_a_number_allocates_nothing_however_long_it_is`; `PropertyTests.The_integer_parser_reads_every_long_and_never_wraps_past_one`, `The_real_parser_reads_the_double_nearest_to_the_decimal`; `ParserTests.Reads_a_number_past_the_largest_real_as_null_and_reports_it`, `Reports_no_number_its_double_holds_however_it_rounds`; `HostileInputTests.Reads_a_number_fifteen_million_digits_long_in_time`, `Quotes_no_number_beyond_what_a_real_can_hold_once_the_diagnostics_are_full` | Property | | An object number or a generation overflowing its cast | The parser builds a header only from a number in 1..`int.MaxValue`, and a reference only from one in 0..`int.MaxValue` — object 0 heads the free list and reads as null —, each with a generation in 0..65,535 (`PdfObjectId.MaxNumber` and `MaxGeneration`, with their reasons), checked before the cast. The index and object streams check theirs the same way, before narrowing: see *Indexing* and *Resolving objects and object streams* | `ParserTests.Refuses_an_object_header_whose_number_or_generation_is_out_of_range`, `Makes_no_reference_of_numbers_an_object_identifier_cannot_hold`; `HostileInputTests.Serves_no_object_under_a_number_past_a_long_that_once_wrapped_to_its_number` | No | @@ -281,9 +281,9 @@ Known gaps: | A missing section tried at many places | Candidates within 512 bytes either side, nearest first, and 32 attempts at most per document (`MaxRelocationCandidates`); a section a guard cut is where it was named, and is tried nowhere else | `CrossReferenceChainTests.Reads_a_section_that_prev_names_a_few_bytes_off`, `Sections_named_a_few_bytes_off_are_looked_for_at_thirty_two_places_a_document_at_most`; `HostileInputTests.Opening_a_chain_of_hybrid_sections_whose_streams_never_close_reads_a_bounded_amount` | Yes | | A classic table of any length | Its window grows ×4 from 64 KB up to `MaxXRefSectionLength`, and the rows read are kept. A table the guard cuts, or whose rows run past a full index, keeps its trailer, found past the rows left unread by a search a megabyte at a time (`ReadTrailerPastRows`): comments are skipped, and anything else no row holds is the section's fault, reported, the search going on to the `trailer` keyword, so that its `/Encrypt` still refuses the document. The search starts at the start of the line the cut fell in, outside any comment and past the table's own keyword; it reads each megabyte once, every byte of it looked at before the next is read, however many comments or stray bytes it holds, and is not made for a place a section may have been moved to, a guess. The chain ends there, since an older section's rows would stand for the objects the unread rows define. What those rows place reads as null until the guard is raised | `ReaderLimitsTests.A_table_whose_keywords_the_bound_cuts_reports_the_bound`, `A_table_its_length_bound_cuts_keeps_its_trailer_and_the_encryption_it_names`, `A_table_its_length_bound_cuts_ends_the_chain_though_its_trailer_names_an_older_section`, `What_ends_the_rows_a_length_bound_cut_is_judged_as_it_is_where_none_did`, `A_table_a_length_bound_cuts_that_runs_to_the_end_of_the_file_has_no_trailer`, `A_table_its_length_bound_cuts_keeps_the_encryption_its_trailer_names_past_what_no_row_holds`, `A_table_cut_before_its_keyword_ends_keeps_its_trailer`, `Rows_past_a_full_index_are_not_read_and_the_trailer_past_them_is`, `A_cut_in_a_comment_among_the_rows_searches_for_the_trailer_from_the_comment_s_start`, `The_trailer_past_rows_left_unread_is_found_wherever_the_search_s_window_ends`, `A_trailer_past_rows_left_unread_that_its_own_bound_cuts_keeps_what_precedes_the_cut`; `DocumentReaderTests.The_search_for_the_trailer_past_rows_left_unread_reads_the_file_once`; `CrossReferenceChainTests.Reads_an_object_a_length_bound_cut_out_of_its_table_as_null_until_the_guard_is_lifted`, `A_section_found_near_where_prev_names_it_is_not_searched_past_the_rows_a_guard_left_unread`; `WindowEdgeTests.A_classic_table_reads_whole_wherever_its_window_edge_falls_near_its_trailer` | No: seeds stay under 120 KB | | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | -| A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s` | Yes | -| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at: the data is taken again where one confirms it, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, which keeps it from being taken again — is the guard's, and not checked against it ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | +| A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_which_no_reader_follows_is_not_read` | Yes | +| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at: the data is taken again where one confirms it, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, which keeps it from being taken again — is the guard's, and not checked against it ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | @@ -348,12 +348,12 @@ Known gaps: | Threat | Defense | Held by | Fuzzed | |---|---|---|---| | A `/Length` that is negative, past `int.MaxValue`, not an integer or unresolvable | Only 0..`int.MaxValue` is taken as a length. Anything else sends the reader to the one search for the stream's first `endstream` (below) | `StreamLengthTests.A_length_that_cannot_be_taken_is_reported_as_the_file_wrote_it` | Partly | -| A `/Length` naming an object a guard cut or left out of the index, judged by what the cut left of it: null, reported as no length, or the `1` it parted of `15`, reported as a length the data contradicts; or the stream's missing `endstream` left unreported along with the length | Keyed on the object the reference leads to, through a chain of references too, never on the value read (`StreamLengthForm.ObjectCut`, `ObjectUnindexed`): an integer read before the cut is only a place to look for `endstream` at, never kept as a length, and the data is otherwise found by its `endstream`. Nothing is said of the length, whose fault is the guard's, reported where it was reached; a missing `endstream` is still the file's, and reported, naming the object at the end of the chain of references, the one the guard cut or left out: "The stream's /Length names object N G, which one of the reader's limits cut, and no endstream follows its data …", or "…, which one of the reader's limits left out of the index, and no endstream follows its data …". A cross-reference stream's `/Length` so cut is left alike once the chain is read, and makes its section the guard's when no `endstream` ends its data, as *Indexing* says ([#224]) | `GuardCutValueTests.A_length_a_guard_cut_is_not_judged_and_the_data_is_found_by_its_endstream`, `A_length_a_guard_cut_leaves_a_missing_endstream_the_file_s_fault`, `A_length_an_object_stream_s_guard_left_unread_is_not_judged`, `A_length_whose_reference_leads_to_an_object_a_guard_left_out_of_the_index_is_not_judged`, `A_missing_endstream_names_the_object_a_guard_cut_at_the_end_of_the_length_s_chain_and_keeps_no_length`, `A_missing_endstream_is_the_file_s_fault_when_a_guard_left_the_length_out_of_the_index`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read` | No | +| A `/Length` naming an object a guard cut or left out of the index, judged by what the cut left of it: null, reported as no length, or the `1` it parted of `15`, reported as a length the data contradicts; or the stream's missing `endstream` left unreported along with the length | Keyed on the object the reference leads to, through a chain of references too, never on the value read (`StreamLengthForm.ObjectCut`, `ObjectUnindexed`): an integer read before the cut is only a place to look for `endstream` at, never kept as a length, and the data is otherwise found by its `endstream`. Nothing is said of the length, whose fault is the guard's, reported where it was reached; a missing `endstream` is still the file's, and reported, naming the object at the end of the chain of references, the one the guard cut or left out: "The stream's /Length names object N G, which one of the reader's limits cut, and no endstream follows its data …", or "…, which one of the reader's limits left out of the index, and no endstream follows its data …". A cross-reference stream's `/Length` so cut is left alike once the chain is read, and makes its section the guard's when no `endstream` ends its data, as *Indexing* says ([#224]) | `GuardCutValueTests.A_length_a_guard_cut_is_not_judged_and_the_data_is_found_by_its_endstream`, `A_length_a_guard_cut_leaves_a_missing_endstream_the_file_s_fault`, `A_length_an_object_stream_s_guard_left_unread_is_not_judged`, `A_length_whose_reference_leads_to_an_object_a_guard_left_out_of_the_index_is_not_judged`, `A_missing_endstream_names_the_object_a_guard_cut_at_the_end_of_the_length_s_chain_and_keeps_no_length`, `A_missing_endstream_is_the_file_s_fault_when_a_guard_left_the_length_out_of_the_index`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`; `GuardCutValueTests.A_length_whose_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `A_missing_endstream_names_the_last_object_of_a_length_s_chain_a_guard_cut_or_left_out_of_the_index` | No | | An indirect `/Length` leading back to its stream, or chained | The loading set and `MaxNestedLoads`, as above | `HostileInputTests.Survives_an_object_whose_length_refers_to_itself`, `Survives_a_chain_of_lengths_longer_than_the_stack_is_deep` | No | | A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | -| A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows` | No | -| Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none` | Property | -| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does: the search goes on past that header up to the next object the index places, and an `endstream` found there ends the data; with none, the header stops it ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.A_stream_a_guard_hid_the_length_of_is_searched_past_a_header_its_data_spells` | Plausibly | +| A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | +| Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | +| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does: the search goes on past that header up to the next object the index places, and an `endstream` found there ends the data; with none, the header stops it ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.A_stream_a_guard_hid_the_length_of_is_searched_past_a_header_its_data_spells`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | | Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, and by white space after a declared length that runs past a first read, about twice the file at most; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | diff --git a/src/AdCodicem.Pdf/IO/EndObjTail.cs b/src/AdCodicem.Pdf/IO/EndObjTail.cs index 589c0525..acba3ac6 100644 --- a/src/AdCodicem.Pdf/IO/EndObjTail.cs +++ b/src/AdCodicem.Pdf/IO/EndObjTail.cs @@ -191,9 +191,11 @@ private int Straddling(ReadOnlySpan bytes, long at, int start, int end) /// token, as the one the data may end at; whether the byte after it does is read next. /// /// - /// A regular character before it makes it part of the token that character is in; a solidus makes it a name, - /// /endobj, and a percent sign part of a comment. The bytes before the keyword are the data's, which no lexing - /// reads: any other delimiter ends a token or stands alone, wherever a string or a dictionary it may open would end. + /// A regular character right before it makes it part of the token that character is in; a solidus makes it a name, + /// /endobj, and a percent sign a comment that starts there, %endobj. Only that byte is read: the bytes + /// before the keyword are the data's, which no lexing reads, so a comment that starts further back on the line does not + /// hide it — any other delimiter ends a token or stands alone, wherever a string, a dictionary or a comment it may open + /// would end. /// /// Whether the keyword starts a token. private bool Found(ReadOnlySpan bytes, long at, int start, long keyword) diff --git a/tests/AdCodicem.Pdf.Tests/ParserTests.cs b/tests/AdCodicem.Pdf.Tests/ParserTests.cs index d8cb4680..7ed5019f 100644 --- a/tests/AdCodicem.Pdf.Tests/ParserTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ParserTests.cs @@ -193,8 +193,8 @@ public void Ends_the_containers_an_object_left_open_at_the_next_object_s_header_ public void Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow(string data, int length) { // Parsed alone, a stream with no endstream ends where its object's endobj ends it, a comment after which is white - // space; a token after it leaves the data running to the next object's header (#174). A name or a comment that the - // keyword would be part of makes it none: "/endobj" is a name. + // space; a token after it leaves the data running to the next object's header (#174). A solidus or a percent sign + // right before the keyword makes it part of a name or of a comment that starts there: "/endobj" is a name. var parser = new PdfObjectParser(Encoding.ASCII.GetBytes("4 0 obj\n<< /Length 200 >>\nstream\n" + data)); parser.TryReadIndirectObject(out _, out var value).Should().BeTrue(); From 213b613eebb6668216c458abca2e22d70f5f762b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:28:05 +0000 Subject: [PATCH 061/146] fix: leave to the guard data a header stops where a guard hid the /Length The search past a header the data spells, for a stream whose /Length a guard cut or left out of the index, took the first endstream it found. An embedded PDF nearly always holds streams of its own: its data was cut at the first inner endstream, in silence. A stream that had lost its endstream took in its own endobj and the unindexed object after it. A rebuild that lifted the guard's hiding could change a stream's data within one document. The second pass is gone. Where a header the data spells stops the search, no endobj of the object's own before it, the length the guard hid decides: the data up to the header is the guard's, cut by it, the object's endobj unjudged, and nothing is said of the file. With no header spelled, a missing endstream stays the file's. The maintainer chose this on 2026-10-23, in place of the search past the header. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- ...-is-readable-and-the-readers-guards-are.md | 5 +- docs/architecture.md | 5 +- docs/threat-model.md | 4 +- docs/website/docs/concepts/diagnostics.md | 15 ++-- docs/website/docs/reference/diagnostics.md | 5 +- docs/website/docs/reference/reader-limits.md | 2 +- .../IO/IPdfStreamDataProvider.cs | 14 ++-- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 81 +++++++------------ src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 31 +++---- src/AdCodicem.Pdf/IO/StreamEndSearch.cs | 6 ++ src/AdCodicem.Pdf/IO/StreamEnds.cs | 16 ++-- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 79 ++++++++++++------ 12 files changed, 144 insertions(+), 119 deletions(-) diff --git a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md index 23f155f2..2d0984c2 100644 --- a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md +++ b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md @@ -98,8 +98,9 @@ bound, reading 21.5 MB to find five bytes and reporting `limit.object`; no windo one search past the window does. What that search reads, with the white space read on after a declared length, is bounded by `EndStreamSearchPasses`, four times the file for a document's searches together, which stays an internal constant with its reason rewritten: a valid file is read past the parser's window only by the search for the data of a -cross-reference stream whose `/Length` the chain cannot read, and by white space and comments after a declared length -that run past a first read — about twice the file at most. `MaxEndStreamGap`, the four bytes of white space accepted +cross-reference stream whose `/Length` the chain cannot read, by the search for the data of a stream whose `/Length` a +guard cut or left out of the index, and by white space and comments after a declared length that run past a first +read — about twice the file at most. `MaxEndStreamGap`, the four bytes of white space accepted between a stream's data and its `endstream`, is removed: more, or a comment, made a right `/Length` a wrong one, and white space and comments are read on as far as they run, counted with the searches — a comment only once white space comes before it, since a `%` right where the length ends is the data's. Comments after the `endobj` that ends a stream diff --git a/docs/architecture.md b/docs/architecture.md index 2629238a..41969915 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -112,8 +112,9 @@ separate types, separate severity scales, and vocabularies that never share a co crafted file can still make it depend on the number a stream is first read under, on the rebuild's scan for trailers, or, once the searches of a document have read four times the file, on which streams were searched first ([#138], M23). A stream whose `/Length` a guard cut or left out of the index may be valid, its length only - hidden: a header its data spells — an embedded file's does — stops its search only when no `endstream` follows - it before the next object the index places ([#224]). With no `endstream` before the stop, the object's `endobj` ends the data where it + hidden: when a header its data spells — an embedded file's does — stops its search, no `endobj` of its own before + it, the data up to the header is the guard's, cut by it, and nothing is said of the file ([#224]). With no + `endstream` before the stop, the object's `endobj` ends the data where it follows the declared length as an `endstream` would, or where it is the last one, a whole keyword — no regular character, solidus or percent sign right before it, which would make it part of a token, a name or a comment that starts there: the data before it, which no lexing reads, says nothing of one further back —, and only diff --git a/docs/threat-model.md b/docs/threat-model.md index e29bc7a1..1c669136 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -353,8 +353,8 @@ Known gaps: | A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | -| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does: the search goes on past that header up to the next object the index places, and an `endstream` found there ends the data; with none, the header stops it ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.A_stream_a_guard_hid_the_length_of_is_searched_past_a_header_its_data_spells`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | -| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, and by white space after a declared length that runs past a first read, about twice the file at most; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | +| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | +| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | | Encoded data of any length: a `/Length` of a few digits claiming gigabytes, one past what an array holds, which would exhaust memory at each read, or data decoding hands back as it is, which `MaxDecodedStreamLength` does not bound | `MaxStreamLength` (*The guards*): what is read of one stream's data at once stops at it, what lies past it is cut by a guard and reported as `limit.stream-length` — once for the stream in its document's diagnostics, and with each decoding that meets the cut in diagnostics a caller passes to `Decode`, as `limit.decoded-stream` is —, and the filters decode what was read as data the reader cut ([#170]) | `ReaderLimitsTests.A_stream_cut_at_MaxStreamLength_decodes_what_lies_within_it_as_data_the_reader_cut_and_is_reported_once`, `Data_decoding_hands_back_as_it_is_is_bounded_by_MaxStreamLength_and_not_by_MaxDecodedStreamLength`, `A_length_past_what_an_array_holds_ends_at_the_guard_and_never_in_an_exhausted_memory`, `Decoding_into_diagnostics_of_one_s_own_reports_the_cut_max_stream_length_made_each_time` | No | diff --git a/docs/website/docs/concepts/diagnostics.md b/docs/website/docs/concepts/diagnostics.md index 51a66cd3..356e5918 100644 --- a/docs/website/docs/concepts/diagnostics.md +++ b/docs/website/docs/concepts/diagnostics.md @@ -68,9 +68,11 @@ null, or `1` of a `15` it parted — is the guard's. An integer read before the `endstream` at, and the data is otherwise found by its `endstream`, as for a stream with no length; nothing is said of the length, while a missing `endstream` is still the file's, and reported. One that names an object a guard left out of the index is not judged either: its data is found the same way, and a missing `endstream` reported as the file's. Such a -stream may be valid, its length only hidden, and its data may spell an object header, as an embedded PDF file's does: -the search goes on past that header to the next object the index places, and an `endstream` found there ends the data, -as the length would have. With none, the header stops it as any other stream's. +stream may be valid, its length only hidden, and its data may spell an object header, as an embedded PDF file's does — +and the `endstream` of a stream the embedded file holds, which no search can tell from the stream's own. A header the +data spells that stops the search, with no `endobj` of the object's own before it, leaves the data up to it to the +guard: it is cut by the guard, as data `MaxStreamLength` cuts is, whether the object ends with `endobj` is not judged, +and nothing is said of the file. A cross-reference stream whose `/Length` the chain cannot read yet, which a valid file may hold, is searched as well, up to the next object the sections read so far place, or the end of the file, and its length read once the chain is. Its @@ -81,14 +83,15 @@ cut: the length that would have said where the data ends is then the guard's, an there. The search is not one of the reader's limits: a valid file's `endstream` follows its length, so only a damaged stream is -searched — and a cross-reference stream whose `/Length` the chain cannot read yet —, each once. Where it stops depends +searched — and a cross-reference stream whose `/Length` the chain cannot read yet, and a stream whose `/Length` a guard +hid —, each once. Where it stops depends on the file's bytes and on its index as the file wrote it, which the reader rebuilding or correcting its own index later does not change: in the 401 documents of the project's test corpus measured, each stream takes the same length whatever was read before it — though a damaged index can still serve another copy of an object, at another offset, according to what was read first. A file crafted for it can still make what a stream holds depend on which object was asked for first. What the searches of one document read together is bounded: none starts once they have read four times the file's -length. For a valid file, what they read — its cross-reference streams searched, the white space after a declared length -read on — comes to about twice the file at most, and the searches of damaged streams read twice the file at most — 0.35 +length. For a valid file, what they read — its cross-reference streams searched, its streams whose length a guard hid, +the white space after a declared length read on — comes to about twice the file at most, and the searches of damaged streams read twice the file at most — 0.35 of it at most, all searches together, in the test corpus — unless their stretches overlap: one object's header inside another's dictionary, or a header the search does not take — a regular character glued before its number — that the index as the file wrote it does not place. Past the bound, only the bytes the parser holds are searched: a declared diff --git a/docs/website/docs/reference/diagnostics.md b/docs/website/docs/reference/diagnostics.md index 71bd966a..5642208e 100644 --- a/docs/website/docs/reference/diagnostics.md +++ b/docs/website/docs/reference/diagnostics.md @@ -185,8 +185,9 @@ cut. A `/Length` that names an object a guard left out of the index, directly or judged either: the data is found by its `endstream` the same way, and a missing one is the file's, the message naming the object the guard left out — `The stream's /Length names object 21 0, which one of the reader's limits left out of the index, and no endstream follows its data`, then where the search stopped and what was taken. For either, an object -header the data spells, as an embedded PDF file's does, stops the search only when no `endstream` follows it before the -next object the index places: one that does ends the data, and nothing is reported of the stream. +header the data spells, as an embedded PDF file's does, that stops the search — no `endobj` of the object's own before +it — leaves the data up to it to the guard: nothing is reported of the stream, whose data is cut by the guard, and +whether its object ends with `endobj` is not judged. ```text Warning stream.length-invalid at 5501: The stream declared 19954 bytes but ended after 19952. diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index 848a4e6f..d745b988 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -172,7 +172,7 @@ the documents producers write. They are not options: |---|---|---| | Nesting of arrays and dictionaries | 128 levels | Real documents nest a handful of levels; the bound keeps a hostile file from exhausting the stack | | A reference that resolves to another reference | followed 32 times | Producers write the object itself; the bound breaks a chain that loops | -| Searches for a stream's `endstream` | four times the file's length, for a document's searches together | A valid stream's `endstream` follows its declared length: the file is read past the bytes an object is parsed through only for the data of a cross-reference stream whose `/Length` the chain cannot read, and for white space and comments after a declared length that run past a first read — about twice the file at most. Past the bound, a stream whose `endstream` the bytes read with its object do not show is not searched for further, and its report says so | +| Searches for a stream's `endstream` | four times the file's length, for a document's searches together | A valid stream's `endstream` follows its declared length: the file is read past the bytes an object is parsed through only for the data of a cross-reference stream whose `/Length` the chain cannot read, for the data of a stream whose `/Length` a guard cut or left out of the index, and for white space and comments after a declared length that run past a first read — about twice the file at most. Past the bound, a stream whose `endstream` the bytes read with its object do not show is not searched for further, and its report says so | | Objects loaded while loading another | 64 | An indirect `/Length` leads to an integer, and a stream resolves its filters and the parameters they read before decoding any of its data: no level holds decoded data while the next loads | | Object numbers | up to 2,147,483,647 | An object's number is an `int`, and ISO 32000-1's Annex C advises at most 8,388,607 indirect objects. A number past it is refused where it is read — a header, a reference, a subsection, a row, an object stream's header —, never narrowed to one that would name another object. An object is numbered from 1; 0 names the head of the free list, where a reference, a subsection or a row may give it | | An object stream's members | each read to the next one's start, the last to the end of the decoded data | ISO 32000-1 (7.5.7) has a header list its objects by increasing offset, so that each lies before the next one starts: a pair whose offset is not past the one before it, or whose object would start past the data, ends the header there, as a stray token does, and the objects listed before it are read. A member past `MaxObjectLength` is that limit's | diff --git a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs index 0b95ecbc..92e1ab3b 100644 --- a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs +++ b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs @@ -16,8 +16,11 @@ internal interface IPdfStreamDataProvider : IPdfFileEdge /// Gets the length of the file the stream data is read from. long SourceLength { get; } - /// Creates stream data for bytes at . - PdfStreamData Create(long absoluteOffset, int length); + /// + /// Creates stream data for bytes at , marked cut by a guard + /// when : a guard hid what would have said where the data ends (#224). + /// + PdfStreamData Create(long absoluteOffset, int length, bool cutByGuard = false); /// /// Gives where the data of a stream starts in the file: past the end-of-line after its stream keyword, which ends @@ -67,12 +70,7 @@ internal interface IPdfStreamDataProvider : IPdfFileEdge /// Whether the stream is a cross-reference stream whose /Length the chain could not read (#182): its rows may /// spell an object header and end in a carriage return, so neither stops nor trims the search (#220). /// - /// - /// Whether the stream's /Length leads to an object a guard cut or left out of the index, which hides where the - /// data of a stream that may be valid ends: a header its data spells stops the search only when no endstream - /// follows it before the next object the index places, or the end of the file (#224). - /// - StreamEndSearch FindEndStream(int number, long absoluteDataStart, long declaredEnd, ReadOnlySpan buffered, bool deferred, bool lengthHidden); + StreamEndSearch FindEndStream(int number, long absoluteDataStart, long declaredEnd, ReadOnlySpan buffered, bool deferred); /// /// Resolves the object a stream's /Length names, and says whether the file holds it and whether it could be diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 49572dea..92cc99a0 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -145,11 +145,13 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// /// /// Not a guard a valid file reaches (ADR 34), argued here once for every read it counts. A valid file is read past the - /// parser's window in two ways only: a cross-reference stream whose /Length the chain cannot read (#220) is - /// searched once to its endstream, and white space after a declared length that runs past a first read is read - /// on once for each place (#174). The data of different sections lies apart, and so do the runs of white space after - /// different streams: together those reads cover the file once, and one read past each — about twice the file at - /// most. Every other search is of an invalid stream, whose endstream does not follow its declared length. Each + /// parser's window in three ways only: a cross-reference stream whose /Length the chain cannot read (#220) is + /// searched once to its endstream; a stream whose /Length a guard cut or left out of the index (#224) is + /// searched once to its endstream, or to a header its data spells, which leaves the data up to it to the guard; + /// and white space after a declared length that runs past a first read is read on once for each place (#174). The data + /// of different streams lies apart, and so do the runs of white space after them: together those reads cover the file + /// once, and one read past each — about twice the file at most. Every other search is of an invalid stream, whose + /// endstream does not follow its declared length. Each /// reads from a stream's data to the first object header the bytes hold after it, or to the next object the index as /// written places, whichever is nearer, and its reads grow from twice what the parser holds of the data /// (): it reads at most about twice what its object spans, and the searches of @@ -374,7 +376,7 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// its /Length was one the chain could not read: the file is searched once for each stream, however many times /// it is parsed. With it, the declared end it last said what follows of, read again only when that end changes. /// - private readonly Dictionary<(long DataStart, bool Deferred, bool LengthHidden), (StreamEndSearch Search, long DeclaredEnd)> _endStreamSearches = []; + private readonly Dictionary<(long DataStart, bool Deferred), (StreamEndSearch Search, long DeclaredEnd)> _endStreamSearches = []; /// /// What follows each place the reader read on past white space and comments to find an endstream (#174): read @@ -1543,12 +1545,19 @@ private long EntryPosition(XRefEntry entry) long IPdfStreamDataProvider.SourceLength => _source.Length; /// - PdfStreamData IPdfStreamDataProvider.Create(long absoluteOffset, int length) + PdfStreamData IPdfStreamDataProvider.Create(long absoluteOffset, int length, bool cutByGuard) { // A declared length is a claim made by the file, so it is clamped to what the file can hold. // Without this, "/Length 2147483647" would ask for two gigabytes of memory. var available = (int)Math.Clamp(_source.Length - absoluteOffset, 0, int.MaxValue); - return new FileStreamData(this, absoluteOffset, Math.Clamp(length, 0, available), _guard); + var data = new FileStreamData(this, absoluteOffset, Math.Clamp(length, 0, available), _guard); + + if (cutByGuard) + { + data.MarkCutByGuard(); + } + + return data; } /// @@ -1781,25 +1790,24 @@ private bool IsEndObjKeywordAt(long position) /// but for a cross-reference stream whose /Length the chain could not read, which is searched with neither /// stop the rule for headers makes nor a carriage return trimmed, as : its rows may spell /// a header, and end in a carriage return; and for a stream whose /Length a guard cut or left out of the index, - /// , whose data may spell a header, as an embedded file's does: the search goes on past - /// it to the next object the index places, and the header stops it only when no endstream follows it there - /// (#224). What it reads is the stretch between the stream and the next object, + /// which a valid file may hold, searched as an invalid one is: a header its data spells — an embedded file's does — + /// leaves the data up to it to the guard (#224). What it reads is the stretch between the stream and the next object, /// once for each stream: the result is kept, and a stream parsed again — after the cache let it go, or a rebuild of /// the index — is not searched again. What the searches read together is bounded by /// . /// /// StreamEndSearch IPdfStreamDataProvider.FindEndStream( - int number, long absoluteDataStart, long declaredEnd, ReadOnlySpan buffered, bool deferred, bool lengthHidden) + int number, long absoluteDataStart, long declaredEnd, ReadOnlySpan buffered, bool deferred) { - var found = _endStreamSearches.TryGetValue((absoluteDataStart, deferred, lengthHidden), out var known); + var found = _endStreamSearches.TryGetValue((absoluteDataStart, deferred), out var known); if (found && known.DeclaredEnd == declaredEnd) { return known.Search; } - var search = found ? known.Search : SearchEndStreamOnce(number, absoluteDataStart, buffered, deferred, lengthHidden); + var search = found ? known.Search : SearchEndStreamOnce(number, absoluteDataStart, buffered, deferred); // Whether the object's endobj follows the declared length depends on the length the stream is read with now, which a // rebuild can change — the /Length an object another definition replaces —: it is read again when that length does, @@ -1813,7 +1821,7 @@ StreamEndSearch IPdfStreamDataProvider.FindEndStream( }; } - _endStreamSearches[(absoluteDataStart, deferred, lengthHidden)] = (search, declaredEnd); + _endStreamSearches[(absoluteDataStart, deferred)] = (search, declaredEnd); return search; } @@ -1822,8 +1830,7 @@ StreamEndSearch IPdfStreamDataProvider.FindEndStream( /// endstream, and reads what its object's endobj does when none is found; what follows the declared length /// is left to . /// - private StreamEndSearch SearchEndStreamOnce( - int number, long absoluteDataStart, ReadOnlySpan buffered, bool deferred, bool lengthHidden) + private StreamEndSearch SearchEndStreamOnce(int number, long absoluteDataStart, ReadOnlySpan buffered, bool deferred) { var nextObject = NextObjectStart(number, absoluteDataStart); @@ -1834,8 +1841,7 @@ private StreamEndSearch SearchEndStreamOnce( absoluteDataStart, buffered, nextObject ?? _source.Length, - stopsAtHeader: !deferred, - keepsCarriageReturn: deferred, + deferred, readsFile, out var header, out var searched, @@ -1845,30 +1851,6 @@ private StreamEndSearch SearchEndStreamOnce( // A header the bytes hold is looked for only before the next object the index places: found, it is the nearer. var stop = header >= 0 ? header : nextObject; - // A /Length a guard cut or left out of the index hides where the data of a stream that may be valid ends: data that - // spells an object header, as an embedded file's does, is searched on past it, up to the next object the index - // places, and an endstream found there ends it as the length would have. With none, the header stops it (#224). - if (dataEnd < 0 && header >= 0 && lengthHidden && _endStreamSearchBytes < EndStreamSearchPasses * _source.Length) - { - var pastHeader = SearchEndStream( - header, - [], - nextObject ?? _source.Length, - stopsAtHeader: false, - keepsCarriageReturn: false, - readsFile: true, - out _, - out _, - out _, - out _); - - if (pastHeader >= 0) - { - dataEnd = pastHeader; - stop = nextObject; - } - } - // Past int.MaxValue bytes, no length the reader holds a stream by reaches the endstream found: only a file of more // than 2 GB can put one there, and the stream is read as one whose endstream is not found. With none, what the // object's endobj does is read once with the search, as long as the searches may read. @@ -1881,6 +1863,7 @@ private StreamEndSearch SearchEndStreamOnce( { EndObjSeen = endObjSeen, ObjectEnd = objectEnd, + HeaderStopped = header >= 0, }; } @@ -2004,8 +1987,7 @@ internal static int EndStreamSearchFirstRead(int buffered) => /// Where the data starts in the file. /// The bytes of the data the parser holds, from its start. /// Where the next object the index places starts, or the end of the file. - /// Whether an object header the bytes hold stops the search. - /// Whether a carriage return before the end-of-line's line feed is data. + /// Whether the stream's /Length is one the chain could not read, which no header stops. /// Whether the file past the bytes the parser holds may be read. /// Where the header that stopped the search starts, or -1. /// Whether the search reached where it stops, rather than the end of what it might read. @@ -2018,8 +2000,7 @@ private long SearchEndStream( long dataStart, ReadOnlySpan buffered, long limit, - bool stopsAtHeader, - bool keepsCarriageReturn, + bool deferred, bool readsFile, out long header, out bool searched, @@ -2035,7 +2016,7 @@ private long SearchEndStream( var inBuffer = (int)Math.Clamp(limit - dataStart, 0, buffered.Length); text.Read(buffered[..inBuffer], dataStart); - var found = StreamEnds.SearchStretch(text, endsRange: dataStart + inBuffer >= limit, stopsAtHeader, keepsCarriageReturn, out header); + var found = StreamEnds.SearchStretch(text, endsRange: dataStart + inBuffer >= limit, deferred, out header); endObjSeen = StreamEnds.HoldsEndObj(buffered[..inBuffer], header >= 0 ? (int)(header - dataStart) : inBuffer); tail.Read(buffered[..inBuffer], dataStart, header >= 0 ? header : dataStart + inBuffer); @@ -2076,7 +2057,7 @@ private long SearchEndStream( } text.Read(span, position); - found = StreamEnds.SearchStretch(text, endsRange: position + span.Length >= end, stopsAtHeader, keepsCarriageReturn, out header); + found = StreamEnds.SearchStretch(text, endsRange: position + span.Length >= end, deferred, out header); endObjSeen |= StreamEnds.HoldsEndObj(span, header >= 0 ? (int)(header - position) : span.Length); tail.Read(span, position, header >= 0 ? header : position + span.Length); @@ -3650,7 +3631,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // file, ends nowhere the section says: its rows are not read (#220). Where the /Length leads to an object a guard cut, // the length that would have said is the guard's, as a value its rows are read with is (#263): the section counts as // cut, the chain ends here, and the guard is reached once the chain is read. - if (deferred is not null && _endStreamSearches.TryGetValue((xrefStream.Data.Position, true, false), out var known) && + if (deferred is not null && _endStreamSearches.TryGetValue((xrefStream.Data.Position, true), out var known) && known.Search is { Length: null } search) { // The fault names the link of the chain that could not be read — the one no section read so far places, or the diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 61e59c27..ac459625 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -730,8 +730,8 @@ private PdfObject ParseDictionaryOrStream(int depth, int openedAt) /// The declared length is confirmed by an endstream after it, white space and comments between, however many. /// Unconfirmed, or with no length to confirm, the data is searched for its first endstream, which ends it: in the /// bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold and the next - /// object the index places, or to the end of the data — past such a header when a guard hid the length, as long as an - /// endstream follows it before that object. Stream data never counts toward the bytes the parser is given + /// object the index places, or to the end of the data. Where a guard hid the length, a header the data spells that + /// stops the search leaves the data up to it to the guard. Stream data never counts toward the bytes the parser is given /// to read an object through: a window is never grown to find an endstream. With none before the stop, the /// object's endobj plays its part: it ends the data where it follows the declared length, or where it ends the /// object before the stop; without it, a declared length that ends before the stop is kept, and the data otherwise @@ -794,18 +794,11 @@ private PdfStream ReadStream(PdfDictionary dictionary) // A cross-reference stream whose /Length the chain could not read keeps a carriage return before the line feed as // data, which it may be — rows can end in 0x0D, and the length, once read, is checked against what was taken —, - // and is not stopped by rows that spell an object header (#182, #220). One whose /Length a guard cut or left out of - // the index is searched on past a header its data spells, which a valid stream's may (#224). + // and is not stopped by rows that spell an object header (#182, #220). var deferred = declared.Form == StreamLengthForm.Deferred; var declaredEnd = length < 0 ? -1 : _baseOffset + dataStart + length; var search = _streamData is not null - ? _streamData.FindEndStream( - _objectNumber, - _baseOffset + dataStart, - declaredEnd, - span[Math.Min(dataStart, span.Length)..], - deferred, - lengthHidden: declared.Form is StreamLengthForm.ObjectCut or StreamLengthForm.ObjectUnindexed) + ? _streamData.FindEndStream(_objectNumber, _baseOffset + dataStart, declaredEnd, span[Math.Min(dataStart, span.Length)..], deferred) : SearchBuffer(span, dataStart, declaredEnd, deferred); if (search is not { } settled) @@ -920,6 +913,15 @@ private PdfStream FinishWithoutEndStream( return Finish(dictionary, dataStart, length, span.Length); } + // A /Length a guard cut or left out of the index hides where the data ends, which a header the data spells stops the + // search at, as a valid stream's may — an embedded file's does —, no endobj of the object's own before it: the data + // up to it is the guard's, cut by it, the object's endobj unjudged, and nothing is said of the file (#224). + if (declared.Form is StreamLengthForm.ObjectCut or StreamLengthForm.ObjectUnindexed && search.HeaderStopped) + { + _endObjUnknown = true; + return Finish(dictionary, dataStart, upToStop, span.Length, cutByGuard: true); + } + // The data runs to the next object, which no declared length passes, or to the end of the file. The object lacks its // endobj, unless what is taken as its data holds one, which may be the object's own: where it ends is then unknown. if (KeepLengthFault(Fault(declared, dataStart, upToStop, null, missing, search.NextObject))) @@ -1004,7 +1006,7 @@ static string Where(long? nextObject) => nextObject is { } at var text = new HeaderText(carried, carriedAt, start); text.Read(span[dataStart..], start); - var dataEnd = StreamEnds.SearchStretch(text, endsRange: _endsData, stopsAtHeader: !deferred, keepsCarriageReturn: deferred, out var header); + var dataEnd = StreamEnds.SearchStretch(text, endsRange: _endsData, deferred, out var header); if (dataEnd >= 0) { @@ -1023,6 +1025,7 @@ static string Where(long? nextObject) => nextObject is { } at return new StreamEndSearch(null, header < 0 ? null : header, EndObjState.Unknown) { + HeaderStopped = header >= 0, EndObjSeen = StreamEnds.HoldsEndObj(stretch, stretch.Length), EndObjAfterDeclared = declaredEnd >= start && declaredEnd - _baseOffset <= stop && StreamEnds.IsEndObjAt(span[..stop], (int)(declaredEnd - _baseOffset), reachesEndOfFile: true) == true, @@ -1213,12 +1216,12 @@ private bool KeepLengthFault(StreamLengthFault fault) private readonly void WarnLengthFault(string code, string message, int dataStart) => _diagnostics!.Warn(code, message, _baseOffset + dataStart); - private PdfStream Finish(PdfDictionary dictionary, int dataStart, int length, int bufferLength) + private PdfStream Finish(PdfDictionary dictionary, int dataStart, int length, int bufferLength, bool cutByGuard = false) { length = Math.Max(length, 0); var data = _streamData is not null - ? _streamData.Create(_baseOffset + dataStart, length) + ? _streamData.Create(_baseOffset + dataStart, length, cutByGuard) : PdfStreamData.FromMemory(_memory.Slice(dataStart, Math.Min(length, bufferLength - dataStart))); var afterData = (int)Math.Min((long)dataStart + length, bufferLength); diff --git a/src/AdCodicem.Pdf/IO/StreamEndSearch.cs b/src/AdCodicem.Pdf/IO/StreamEndSearch.cs index 48f28739..0d5debdd 100644 --- a/src/AdCodicem.Pdf/IO/StreamEndSearch.cs +++ b/src/AdCodicem.Pdf/IO/StreamEndSearch.cs @@ -39,4 +39,10 @@ internal readonly record struct StreamEndSearch(int? Length, long? NextObject, E /// unknown. /// public bool EndObjSeen { get; init; } + + /// + /// Gets a value indicating whether, no endstream found, an object header the bytes hold stopped the search, which + /// then gives, rather than the next object the index places or the end of the file. + /// + public bool HeaderStopped { get; init; } } diff --git a/src/AdCodicem.Pdf/IO/StreamEnds.cs b/src/AdCodicem.Pdf/IO/StreamEnds.cs index 51e8bd9c..258f82d4 100644 --- a/src/AdCodicem.Pdf/IO/StreamEnds.cs +++ b/src/AdCodicem.Pdf/IO/StreamEnds.cs @@ -114,21 +114,19 @@ public static bool HoldsEndObj(ReadOnlySpan bytes, int before) => /// /// /// The overlap between reads puts the end-of-line before a keyword first seen in a read inside it, unless the keyword - /// starts the data. A cross-reference stream whose /Length the chain could not read is searched neither stopped - /// by a header nor trimmed of a carriage return: its rows may spell a header, and may end in a carriage return (#220). - /// A stream whose /Length a guard hid is searched on past a header its data spells, once the header stopped it - /// (#224). + /// starts the data. A cross-reference stream whose /Length the chain could not read is searched as + /// : its rows may spell a header, and may end in a carriage return, so neither stops nor + /// trims it (#220). /// /// The bytes read, after what the reads before carry into them. /// Whether they end where the search does: at the next object the index places, or the end of the file. - /// Whether an object header the bytes hold before the keyword stops the search. - /// Whether a carriage return before the end-of-line's line feed is data. + /// Whether the stream's /Length is one the chain could not read, which is no fault. /// Where the header that stopped the search starts in the file, or -1. - public static long SearchStretch(in HeaderText text, bool endsRange, bool stopsAtHeader, bool keepsCarriageReturn, out long header) + public static long SearchStretch(in HeaderText text, bool endsRange, bool deferred, out long header) { var span = text.Bytes; var keyword = span.IndexOf(EndStreamKeyword); - header = stopsAtHeader ? text.FindHeader(keyword >= 0 ? keyword : span.Length, endsRange) : -1; + header = deferred ? -1 : text.FindHeader(keyword >= 0 ? keyword : span.Length, endsRange); if (keyword < 0 || header >= 0) { @@ -140,7 +138,7 @@ public static long SearchStretch(in HeaderText text, bool endsRange, bool stopsA keyword--; } - if (!keepsCarriageReturn && keyword > 0 && span[keyword - 1] == (byte)'\r') + if (!deferred && keyword > 0 && span[keyword - 1] == (byte)'\r') { keyword--; } diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 18bb8707..5361ad84 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -184,24 +184,31 @@ public void A_missing_endstream_is_the_file_s_fault_when_a_guard_left_the_length } [Theory] - [InlineData("cut", true)] - [InlineData("left out", true)] - [InlineData("cut", false)] - [InlineData("left out", false)] - public void A_stream_a_guard_hid_the_length_of_is_searched_past_a_header_its_data_spells(string guard, bool endStream) + [InlineData("cut", "endstream")] + [InlineData("left out", "endstream")] + [InlineData("cut", "inner stream")] + [InlineData("left out", "inner stream")] + [InlineData("cut", "none")] + [InlineData("left out", "none")] + public void Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s(string guard, string end) { - // Object 4 embeds a PDF file, whose data spells "1 0 obj", and its /Length, object 10, is one MaxObjectLength cuts or - // MaxIndexEntries leaves out of the index: searched for its endstream, the data stopped at the header it spells, and - // the file was said to lack the endstream it holds (#224). Without one, the header still stops it, the file's fault. - const string Embedded = "%PDF-1.4\n1 0 obj\n<< /Type /Catalog >>\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF"; + // Object 4 embeds a PDF file, whose data spells "1 0 obj" — and holds a stream of its own, whose endstream comes + // after —, and its /Length, object 10, is one MaxObjectLength cuts or MaxIndexEntries leaves out of the index. The + // search for its endstream stops at the header the data spells: the file was said to lack the endstream it holds, + // or the data was taken to the embedded stream's. What decides is the length the guard hid: the data up to the + // header is the guard's, cut by it, the object's endobj unjudged, and nothing said of the file, whatever follows + // (#224). + var embedded = "%PDF-1.4\n1 0 obj\n<< /Type /Catalog >>\nendobj\n" + + (end == "inner stream" ? "2 0 obj\n<< /Length 5 >>\nstream\nhello\nendstream\nendobj\n" : string.Empty) + + "trailer\n<< /Root 1 0 R >>\n%%EOF"; var file = new HandWrittenFile(); var rows = new Dictionary { [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), - [4] = file.Object(4, "<< /Type /EmbeddedFile /Length 10 0 R >>\nstream\n" + Embedded + (endStream ? "\nendstream" : string.Empty)), + [4] = file.Object(4, "<< /Type /EmbeddedFile /Length 10 0 R >>\nstream\n" + embedded + (end == "none" ? string.Empty : "\nendstream")), [6] = file.Object(6, "(six)"), - [10] = file.Object(10, (guard == "cut" ? new string(' ', 2000) : string.Empty) + Embedded.Length.ToString(CultureInfo.InvariantCulture)), + [10] = file.Object(10, (guard == "cut" ? new string(' ', 2000) : string.Empty) + embedded.Length.ToString(CultureInfo.InvariantCulture)), }; file.Table(11, rows, "/Size 11 /Root 1 0 R"); var limits = guard == "cut" @@ -209,20 +216,46 @@ public void A_stream_a_guard_hid_the_length_of_is_searched_past_a_header_its_dat : PdfReaderLimits.Default with { MaxIndexEntries = 8 }; using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = limits }); - var data = document.GetObject(new PdfObjectId(4)).AsStream().Required().GetRawBytes().ToArray(); - var codes = document.Diagnostics.Select(entry => entry.Code).ToArray(); + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); - if (endStream) - { - data.Should().Equal(Encoding.ASCII.GetBytes(Embedded)); - codes.Should().Equal(guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries); - document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); - } - else + stream.GetRawBytes().ToArray().Should().Equal("%PDF-1.4\n"u8.ToArray()); + stream.Data.CutByGuard.Should().BeTrue(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal( + guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries); + document.Reader.TryGetStreamLengthFault(4, out _).Should().BeFalse(); + document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); + new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.ObjectEndObjMissing).Should().BeFalse(); + } + + [Theory] + [InlineData("cut")] + [InlineData("left out")] + public void A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header(string guard) + { + // Object 4, whose /Length a guard hid, lost its endstream; its own endobj, then an object no row places, a stream + // with an endstream, follow: the header that object starts with is no header the data spells, and the endobj before + // it ends the data, the missing endstream the file's (#224). + var file = new HandWrittenFile(); + var rows = new Dictionary { - data.Should().Equal("%PDF-1.4\n"u8.ToArray()); - codes.Should().Contain(PdfDiagnosticCodes.StreamLengthInvalid); - } + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + [4] = file.Object(4, "<< /Length 10 0 R >>\nstream\nABC"), + }; + file.Object(30, "<< /Length 3 >>\nstream\nxyz\nendstream"); + rows[6] = file.Object(6, "(six)"); + rows[10] = file.Object(10, (guard == "cut" ? new string(' ', 2000) : string.Empty) + "3"); + file.Table(11, rows, "/Size 11 /Root 1 0 R"); + var limits = guard == "cut" + ? PdfReaderLimits.Default with { MaxObjectLength = 1024 } + : PdfReaderLimits.Default with { MaxIndexEntries = 8 }; + using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = limits }); + + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + stream.GetRawBytes().ToArray().Should().Equal("ABC"u8.ToArray()); + stream.Data.CutByGuard.Should().BeFalse(); + document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.StreamTruncated); } [Theory] From d13f47828f7410c506497330204115db3a0d9bc7 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:36:15 +0000 Subject: [PATCH 062/146] fix: read once what a guard's cut left of an object the chain asks again The integer a guard's cut left of a cross-reference stream's /Length object is where its data is taken again to. Each section that asked read the object again from the file, through windows grown to MaxObjectLength: a hundred sections whose /Length names one object of 2 MB that a newer table places read 8.4 MB to open a file of 2 MB, and a thousand, over 1,200 times a file of 17 MB. What the cut left is now kept with the object while the chain is read, and the file read once. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 21 +++++----- .../AdCodicem.Pdf.Tests/HostileInputTests.cs | 42 +++++++++++++++++++ 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 92cc99a0..ae0361fb 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -277,10 +277,11 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// /// The objects could not read while the chain is read because they run past - /// , or null while none did: what they hold is the guard's to hide. Let go - /// with . + /// , with the integer the cut left of each, or null, or null while none did: + /// what they hold is the guard's to hide, and the integer only a place to look for endstream at (#224), read once + /// however many sections ask. Let go with . /// - private HashSet? _cutWhileChainIsRead; + private Dictionary? _cutWhileChainIsRead; /// /// The objects a guard cut that a cross-reference stream's values led to, which the chain could not read and ended at: @@ -750,7 +751,7 @@ public PdfObject GetObject(PdfObjectId id) if (pastGuard) { - (_cutWhileChainIsRead ??= []).Add(id); + (_cutWhileChainIsRead ??= [])[id] = value.AsInteger() is null ? PdfNull.Instance : value; } return read; @@ -2416,7 +2417,7 @@ private bool LeadsPastIndex(PdfObject? value, out PdfObject? read) var number = reference.Id.Number; read = null; return number > 0 && _refusedNumbers?.Contains(number) != true && - (_cutWhileChainIsRead?.Contains(reference.Id) == true || (!_xref.TryGet(number, out _) && MayBeLeftUnread(number))); + (_cutWhileChainIsRead?.ContainsKey(reference.Id) == true || (!_xref.TryGet(number, out _) && MayBeLeftUnread(number))); } } @@ -3982,14 +3983,14 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect Follow(PdfName.Length, written, entry: true, out _, out var cut, links); lengthCut = cut is not null; - // The object the guard cut is one a section read so far places, which ReadWithoutLoading read there. - if (cut is not { } id || !_xref.TryGet(id.Number, out var entry)) + // What the cut left of the object, which ReadWithoutLoading read where a section read so far places it, is kept with + // it: it is not read again for each section that asks. + if (cut is not { } id) { return null; } - TryReadUnrecorded(id, PositionOf(entry), out var value, out _); - var whole = TakeAgain(stream, deferred, value); + var whole = TakeAgain(stream, deferred, _cutWhileChainIsRead![id]); lengthCut = whole is null; return whole; } @@ -4121,7 +4122,7 @@ public bool TryRead(PdfName key, PdfObject? written, bool entry, out PdfObject? // An object a section read so far places, which runs past MaxObjectLength: the guard's, not the file's. var place = "which no section read before it places where it can be read"; - if (_cutWhileChainIsRead?.Contains(reference.Id) == true) + if (_cutWhileChainIsRead?.ContainsKey(reference.Id) == true) { cut = reference.Id; place = "which one of the reader's limits cut"; diff --git a/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs b/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs index 84116851..0351b6c7 100644 --- a/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs +++ b/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs @@ -277,6 +277,48 @@ public void A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_th document.Reader.IsEndObjMissing(5, out _).Should().BeTrue(); } + [Fact] + public void Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once() + { + // A hundred cross-reference streams, chained by /Prev, each with /Length 5 0 R and rows that spell an endstream + // first; the newest table places object 5, which holds 28 then 2,000,000 spaces, past MaxObjectLength. Each section + // takes its data again to the 28 the cut left, which an endstream confirms, and the chain goes on: the cut object + // was read again for each section, and opening read the file several times over (#224). + const int Sections = 100; + var file = new HandWrittenFile(version: "1.5"); + var one = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + var two = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"); + var five = file.Object(5, "28" + new string(' ', 2_000_000)); + var rows = Row(1, one) + Row(1, two); + long previous = -1; + + for (var k = 0; k < Sections; k++) + { + var at = file.Position; + var prev = previous < 0 ? string.Empty : string.Create(CultureInfo.InvariantCulture, $" /Prev {previous}"); + file.Raw(string.Create(CultureInfo.InvariantCulture, $"{100 + k} 0 obj\n<< /Type /XRef /Size 200 /W [1 4 2] /Index [8 2 1 2] /Root 1 0 R /Length 5 0 R{prev} >>\nstream\n")); + file.Raw("\nendstream\0\0\0\0" + rows + "\nendstream\nendobj\n"); + previous = at; + } + + var table = file.Position; + file.Raw(string.Create( + CultureInfo.InvariantCulture, + $"xref\n0 1\n0000000000 65535 f\r\n5 1\n{five:D10} 00000 n\r\ntrailer\n<< /Size 200 /Root 1 0 R /Prev {previous} >>\nstartxref\n{table}\n%%EOF\n")); + var bytes = file.Bytes; + using var source = new StrictCountingSource(bytes); + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = 64 * 1024 } }; + + using var document = PdfDocument.Open(source, options, ownsSource: false); + + document.Reader.Structure.Sections.Should().HaveCount(Sections + 1).And.OnlyContain(section => !section.CutByLimit); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + source.BytesRead.Should().BeLessThan(bytes.Length); + + static string Row(int type, long offset) => string.Concat( + (char)type, (char)((offset >> 24) & 0xFF), (char)((offset >> 16) & 0xFF), (char)((offset >> 8) & 0xFF), (char)(offset & 0xFF), '\0', '\0'); + } + [Fact] public void A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times() { From 95cac5ed99cace2109aa05cf60a70be49e5de81e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:38:32 +0000 Subject: [PATCH 063/146] fix: leave to the guard the rows data taken again to a cut integer still lacks A cross-reference stream's data taken again to the integer a guard's cut left of its /Length, which an endstream its rows spell confirmed, was judged against the section's rows as a length the file gave: a /Length of 210 that MaxObjectLength parted after its 21 left the section three rows of thirty, which were charged to the file, the section incomplete and the index rebuilt. That integer is only a place to look at, the length that decides the guard's: the rows the data taken again still lacks are the guard's, the section cut and the chain ended there. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 2 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 21 ++++++------- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 30 +++++++++++++++++++ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 24 +++++++++++++++ 4 files changed, 66 insertions(+), 11 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index 1c669136..b0901e00 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -283,7 +283,7 @@ Known gaps: | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_which_no_reader_follows_is_not_read` | Yes | | What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at: the data is taken again where one confirms it, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, which keeps it from being taken again — is the guard's, and not checked against it ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, which keeps it from being taken again — is the guard's, and not checked against it ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index ae0361fb..4d58e1db 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3784,7 +3784,6 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect xrefStream = whole; deferred = unsettled with { Taken = whole.Data.Length }; retaken = true; - lengthCut = false; continue; } } @@ -3881,10 +3880,11 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect /// reserved type is ignored there, as the specification says (). The object is read /// where that row places it as reads one: nothing is corrected, cached, recorded or /// reported. One a guard cuts gives what the cut left of it, an integer read before the cut only a place to look for - /// endstream at (#224), and says so when no endstream confirms it. So does the - /// object a guard cut at the end of a chain of references from the object read, or from the /Length when a newer - /// section gave the number: the integer the cut left of that object, read where the sections read so far place it, is - /// the place to look at then (). A row that does not stand otherwise — a newer section gave + /// endstream at (#224), and says so, whether an endstream confirms it or not: + /// rows the data taken again still lacks are the guard's. So does the object a guard cut at the end of a chain of + /// references from the object read, or from the /Length when a newer section gave the number: the integer the cut + /// left of that object, read where the sections read so far place it, is the place to look at then + /// (). A row that does not stand otherwise — a newer section gave /// the number, or refused it; the row lies past those held, places the object in an object stream, or under another /// generation — leaves the data as it was taken. /// @@ -3947,13 +3947,15 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect var read = TryReadUnrecorded(deferred.Length, PositionOf((long)offset), out var value, out var pastGuard); + // Data taken again to an integer a guard's cut left is no length the file gave: rows it still lacks are the guard's. + lengthCut = pastGuard; + if ((read || pastGuard) && TakeAgain(stream, deferred, value) is { } whole) { return whole; } // The object read holds the chain's second link: the chain goes on as far as the parse resolves a /Length through. - lengthCut = pastGuard; return !pastGuard && value is PdfReference ? TakeAgainAtCut(stream, deferred, value, LengthLinks - 1, out lengthCut) : null; } @@ -3976,7 +3978,8 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect /// Gives cross-reference stream with its data taken again, when , read /// through the sections read so far and references, leads to an object a guard cut: to the /// integer the cut left of it, read where they place it, when an endstream confirms it (#224). Null otherwise, - /// and says whether a guard cut that object, which leaves how far the data runs unknown. + /// and says whether a guard cut that object, which leaves how far the data runs unknown, the + /// data taken again or not. /// private PdfStream? TakeAgainAtCut(PdfStream stream, DeferredLength deferred, PdfObject? written, int links, out bool lengthCut) { @@ -3990,9 +3993,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect return null; } - var whole = TakeAgain(stream, deferred, _cutWhileChainIsRead![id]); - lengthCut = whole is null; - return whole; + return TakeAgain(stream, deferred, _cutWhileChainIsRead![id]); } /// diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index cec473ba..2ee866e9 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -490,6 +490,36 @@ public static byte[] UpdatePlacingAPaddedLength( return file.ToArray(); } + /// + /// A file of two revisions. The first holds the catalog, the page tree and the page, object 5, which holds the data's + /// length, 210, after 1,014 spaces — a MaxObjectLength of 1,024 parts it after its 21 —, and cross-reference + /// stream 7, whose /Length names object 5 and whose rows spell endstream after an end-of-line twice, 7 and + /// 21 bytes in, before the rows of objects 1 to 3, 5 and 7 and twenty free ones. The second is a classic table that + /// indexes object 5 again, where it is, and names stream 7 through /Prev. The file is valid. + /// + public static byte[] UpdatePlacingAPartedLength() + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + file.Object(5, new string(' ', 1014) + "210"); + var stream = file.Position; + var spelled = "\nendstream\0\0\0\0"u8.ToArray(); + var data = new List(Encode([(0, 0, 65535)], Rows.Raw)); + data.AddRange(spelled); + data.AddRange(spelled); + data.AddRange(Encode([(1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), (1, file.OffsetOf(5), 0), (1, stream, 0)], Rows.Raw)); + data.AddRange(Encode([.. Enumerable.Repeat((0, 0L, 0), 20)], Rows.Raw)); + file.Stream(7, "/Type /XRef /Size 50 /W [1 4 2] /Index [0 1 20 2 22 2 1 3 5 1 7 1 30 20] /Root 1 0 R /Length 5 0 R", [.. data]); + file.Text("%" + new string('-', 1200) + "\n"); + var xref = file.Position; + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"xref\n0 1\n0000000000 65535 f\r\n5 1\n{file.OffsetOf(5):D10} 00000 n\r\ntrailer\n<< /Size 50 /Root 1 0 R /Prev {stream} >>\nstartxref\n{xref}\n%%EOF\n")); + return file.ToArray(); + } + /// /// A file as writes it, the rows that spell endstream numbering objects 8 /// and 9, but for a length a chain of objects gives: stream 7's /Length names object 5 diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 5361ad84..bea5beee 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -603,6 +603,30 @@ public void The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_whe (document.GetObject(new PdfObjectId(3)).AsDictionary() is not null).Should().Be(lengthOff == 0); } + [Fact] + public void Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s() + { + // MaxObjectLength parts stream 7's /Length, 210, after its 21, which the rows' second spelled endstream confirms: the + // data taken again to it holds 3 rows of the 30 the section declares, and the 27 it lacks were the file's, the + // section incomplete and the index rebuilt. The integer is only a place to look at, the length that decides the + // guard's: what it leaves unread is the guard's (#224). + var file = ChainFiles.UpdatePlacingAPartedLength(); + + using (var whole = PdfDocument.Open(file)) + { + whole.Diagnostics.Should().BeEmpty(); + whole.GetObject(new PdfObjectId(3)).AsDictionary().Should().NotBeNull(); + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.Reader.Structure.Sections[1].CutByLimit.Should().BeTrue(); + document.Reader.Structure.Sections[1].Incomplete.Should().BeFalse(); + new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().BeFalse(); + } + [Fact] public void Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value() { From 3db561cd6daa10cadd8ba07f1b18ff58bd995994 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:43:39 +0000 Subject: [PATCH 064/146] fix: drop what decoding met beside a guard that kept deferred data from being taken again A cross-reference stream whose /Length the chain cannot read is decoded first as taken up to the first endstream, which its rows may spell, and its reports dropped once the data is taken again to its length. A guard on decoding, MaxFilterChainLength as MaxDecodedStreamLength, keeps it from being taken again: the first decoding's other reports stood, a Flate run over data cut short at the spelled endstream reported as the file's fault where the whole read reports nothing. Only the guard's report now stands, once however often the stream is decoded again. Refs #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 2 +- .../Diagnostics/PdfDiagnostics.cs | 37 +++++++++++++++++++ src/AdCodicem.Pdf/IO/PdfFileReader.cs | 8 ++++ tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 16 ++++++-- .../DocumentReaderTests.cs | 20 ++++++++++ 5 files changed, 78 insertions(+), 5 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index b0901e00..1fbb6105 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -283,7 +283,7 @@ Known gaps: | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_which_no_reader_follows_is_not_read` | Yes | | What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, which keeps it from being taken again — is the guard's, and not checked against it ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it, and what decoding it met beside the guard dropped, a filter run over data a spelled `endstream` cut short ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | diff --git a/src/AdCodicem.Pdf/Diagnostics/PdfDiagnostics.cs b/src/AdCodicem.Pdf/Diagnostics/PdfDiagnostics.cs index 974e0899..68888d1a 100644 --- a/src/AdCodicem.Pdf/Diagnostics/PdfDiagnostics.cs +++ b/src/AdCodicem.Pdf/Diagnostics/PdfDiagnostics.cs @@ -129,6 +129,43 @@ internal void RollBack(PdfDiagnosticsMark mark) } } + /// + /// Drops every entry recorded since but the guards reached, their limit.* reports and + /// their count standing: what a reading a guard stopped short met beside the guard is no fault the file holds. The + /// reports decodings moved in that the mark recorded are held again, with the guards' kept, so that a decoding that + /// meets one again does not report it twice. + /// + internal void RollBackBesideLimits(PdfDiagnosticsMark mark) + { + var kept = mark.Count; + var held = mark.DecodingReports is { } recorded ? new Dictionary(recorded) : null; + + for (var i = mark.Count; i < _entries.Count; i++) + { + var entry = _entries[i]; + + if (!entry.Code.StartsWith("limit.", StringComparison.Ordinal)) + { + continue; + } + + _entries[kept++] = entry; + + if (held is not null && _decodingReports?.ContainsKey(entry) == true) + { + held[entry] = held.GetValueOrDefault(entry) + 1; + } + } + + _entries.RemoveRange(kept, _entries.Count - kept); + _suppressed = mark.Suppressed; + + if (held is not null) + { + _decodingReports = held; + } + } + /// /// Moves every entry recorded since into , which applies /// its own capacity, and drops them from this instance. diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 4d58e1db..449d7713 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3791,6 +3791,14 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect break; } + // Data a /Length the chain could not read left to an endstream nothing confirmed, which a guard on decoding kept from + // being taken again to its length: what decoding it met beside the guard — a filter run over data the endstream its + // rows spell cut short — is no fault the file holds, and only the guard's report stands (#220). + if (deferred is not null && !retaken && !decodedWhole) + { + _diagnostics.RollBackBesideLimits(decoding); + } + section.CutByLimit |= !decodedWhole; var position = 0; diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 2ee866e9..113688db 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -192,9 +192,10 @@ byte[] Data(long lengthAt, long streamAt) /// in a row before two rows of reserved types that spell endstream after an end-of-line; the page's row comes /// after them. With , the /Index numbers object 4 twice: first in a row of a /// reserved type, which the specification says to ignore, then in the row that places it. With , - /// the data is compressed in stored blocks, which show the rows as written. The file is valid. + /// the data is compressed in stored blocks, which show the rows as written, times over, each + /// layer a Flate filter. The file is valid. /// - public static byte[] AloneWithIndirectLengthBeforeSpelledRows(bool reservedRowFirst, bool stored = false) + public static byte[] AloneWithIndirectLengthBeforeSpelledRows(bool reservedRowFirst, bool stored = false, int layers = 1) { using var file = new Writer("1.5"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); @@ -208,13 +209,20 @@ byte[] Data(long four, long stream) var data = new List(Encode([(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), .. reserved, (1, four, 0), (1, stream, 0)], Rows.Raw)); data.AddRange("\nendstream\0\0\0\0"u8.ToArray()); data.AddRange(Encode([(1, file.OffsetOf(3), 0)], Rows.Raw)); - return stored ? Compress([.. data]) : [.. data]; + byte[] bytes = [.. data]; + + for (var layer = 0; stored && layer < layers; layer++) + { + bytes = Compress(bytes); + } + + return bytes; } file.Object(4, Data(0, 0).Length.ToString(CultureInfo.InvariantCulture)); var stream = file.Position; var index = reservedRowFirst ? "0 3 4 1 4 2 6 2 3 1" : "0 3 4 2 6 2 3 1"; - var filter = stored ? " /Filter /FlateDecode" : string.Empty; + var filter = stored ? " /Filter [" + string.Join(' ', Enumerable.Repeat("/FlateDecode", layers)) + "]" : string.Empty; file.Stream(5, $"/Type /XRef /Size 8 /W [1 4 2] /Index [{index}]{filter} /Root 1 0 R /Length 4 0 R", Data(file.OffsetOf(4), stream)); file.StartXRef(stream); return file.ToArray(); diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 219f7457..876176c0 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -1923,6 +1923,26 @@ public void A_deferred_length_is_not_checked_against_data_a_guard_cut(bool decod whole.Diagnostics.Should().BeEmpty(); } + [Fact] + public void A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault() + { + // The stream's data, taken up to the endstream its rows spell, is compressed twice in stored blocks, and + // MaxFilterChainLength lets the first Flate alone run: over data cut short at the spelled endstream, it failed, and + // the guard, which kept the data from being taken again to its length, left that failure to the file (#220). + var file = ChainFiles.AloneWithIndirectLengthBeforeSpelledRows(reservedRowFirst: false, stored: true, layers: 2); + using var whole = PdfDocument.Open(file); + using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxFilterChainLength = 1 } }); + + whole.Diagnostics.Should().BeEmpty(); + document.Diagnostics.Should().Contain(entry => entry.Code == PdfDiagnosticCodes.LimitFilterChain); + document.Diagnostics.Should().NotContain(entry => entry.Code.StartsWith("filter.", StringComparison.Ordinal)); + + // The guard's report stands once, however often the stream is decoded again. + document.GetObject(new PdfObjectId(5)).AsStream().Required().Decode(); + new PdfValidator().Validate(document); + document.Diagnostics.Count(entry => entry.Code == PdfDiagnosticCodes.LimitFilterChain).Should().Be(1); + } + [Fact] public void A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is() { From e96f393dde289d7eb24a4b46e64bc1e04cdd67b9 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:48:08 +0000 Subject: [PATCH 065/146] fix: ask the file of every endobj a window ends exactly with An object whose endobj ends exactly at MaxObjectLength's bound reads whole when the endobj follows its value: the file says no regular character goes on. An endobj that ends a container the object left open, or that is all an empty object holds, was not asked: the object was taken for the guard's cut, by a load as by the chain, which lost a revision through a /Prev or a /W so written. Every endobj a window's edge reaches is now asked of the file, which backs the parser. The chain's test drops its /Length row, which read whole without the file's answer, the data taken again to what the cut left: it held nothing of the edge. The retake's own tests hold it. Refs #174, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 21 ++++--- tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs | 63 +++++++++++++++++-- 2 files changed, 69 insertions(+), 15 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index ac459625..86723fef 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -307,11 +307,8 @@ public bool TryReadIndirectObject(out PdfObjectId id, out PdfObject value) var unseen = next.Kind == PdfTokenKind.EndOfInput || next.End >= _memory.Length || _headerUndecided; // What follows a stream whose data runs past the buffer lies past it too: the file was asked at the stream's - // end, and its answer stands where the buffer has none (#55). An endobj a window's edge reaches may be the start of - // a longer keyword: the file says whether a regular character follows it, so that an object a window ends exactly - // with reads whole, as under a larger window; a parser no file backs asks for a larger buffer (#174). - var endObj = IsEndObj(next) || - (next.IsKeyword("endobj"u8) && !PastWindowEdge && _fileEdge is { } file && file.TokenEndsAt(_baseOffset + next.End)); + // end, and its answer stands where the buffer has none (#55). + var endObj = IsEndObj(next); _endObj = endObj ? EndObjState.Present : _endObjUnknown ? EndObjState.Unknown @@ -340,8 +337,8 @@ public bool TryReadIndirectObject(out PdfObjectId id, out PdfObject value) private PdfObject ParseValue(PdfToken token, int depth) { // A value that reaches the end of the buffer may go on past it: "12" of "1234", a string without - // its closing delimiter, the "<" of a "<<". - if (token.Kind != PdfTokenKind.EndOfInput && token.End >= _memory.Length) + // its closing delimiter, the "<" of a "<<". An endobj the file says ends there does not: an empty object's. + if (token.Kind != PdfTokenKind.EndOfInput && token.End >= _memory.Length && !IsEndObj(token)) { _truncated = true; } @@ -414,7 +411,7 @@ private PdfObject ParseValue(PdfToken token, int depth) return PdfNull.Instance; default: - if (!AtWindowEdge(token)) + if (!AtWindowEdge(token) || IsEndObj(token)) { Report(PdfDiagnosticCodes.SyntaxUnexpectedToken, "A token was found where a value was expected.", token.Start); } @@ -1474,9 +1471,13 @@ private void ReportCut( /// /// Determines whether is endobj, and whole: one that reaches the end of a buffer that is - /// a window's edge may be the start of a longer keyword. + /// a window's edge may be the start of a longer keyword, which the file says when it backs the parser, so that an object + /// a window ends exactly with reads whole, as under a larger window, wherever its endobj comes — after its value, where a + /// container it left open is, or as all it holds (#174); a parser no file backs asks for a larger buffer. /// - private readonly bool IsEndObj(PdfToken token) => token.IsKeyword("endobj"u8) && !AtWindowEdge(token); + private readonly bool IsEndObj(PdfToken token) => + token.IsKeyword("endobj"u8) && + (!AtWindowEdge(token) || (!PastWindowEdge && _fileEdge is { } file && file.TokenEndsAt(_baseOffset + token.End))); /// /// Determines whether , read where a key is expected, starts the next object's header diff --git a/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs b/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs index dc8818ed..7ca01538 100644 --- a/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs +++ b/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs @@ -407,18 +407,22 @@ public void An_object_exactly_as_long_as_MaxObjectLength_reads_whole(string body [InlineData("Prev", 9)] [InlineData("XRefStm", 6)] [InlineData("W", 5)] - [InlineData("Length", 5)] + [InlineData("W unclosed", 5)] + [InlineData("Prev empty", 9)] public void An_object_the_chain_reads_exactly_as_long_as_MaxObjectLength_reads_whole(string key, int number) { - // The chain reads object N — a /Prev, an /XRefStm, a cross-reference stream's /W or its /Length — through a window - // MaxObjectLength ends right after its endobj: the file says no regular character goes on past it, and the object is - // whole, as a load reads it. It ended the chain for the guard, which no load then reached (#174, #263). + // The chain reads object N — a /Prev, an /XRefStm, a cross-reference stream's /W — through a window MaxObjectLength + // ends right after its endobj: the file says no regular character goes on past it, and the object is whole, as a + // load reads it, whether the endobj follows its value, ends an array it left open, or is all it holds. It ended the + // chain for the guard, which no load then reached (#174, #263). var file = key switch { "Prev" => ChainFiles.UpdateNamingItsPrevious("9 0 R", padding: 200), "XRefStm" => ChainFiles.HybridNamingItsStream("6 0 R", padding: 200), "W" => ChainFiles.UnderAnUpdate("W", placed: true, padding: 200), - _ => ChainFiles.UpdatePlacingAPaddedLength(chained: false, padding: 200), + "W unclosed" => ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W 5 0 R /Length {length}", "[1 4 2" + new string(' ', 200), compressed: false, predicted: false, placed: true), + _ => Blanked(ChainFiles.UpdateNamingItsPrevious("9 0 R", padding: 200)), }; var text = Encoding.Latin1.GetString(file); var header = text.IndexOf(string.Create(CultureInfo.InvariantCulture, $"\n{number} 0 obj\n"), StringComparison.Ordinal) + 1; @@ -440,6 +444,55 @@ public void An_object_the_chain_reads_exactly_as_long_as_MaxObjectLength_reads_w var throwing = () => PdfDocument.Open( file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = limit }, ThrowOnLimit = true }).Dispose(); throwing.Should().NotThrow(); + + // Object 9's value, the first table's offset, given to spaces: the object holds nothing but its endobj. + static byte[] Blanked(byte[] file) + { + var text = Encoding.Latin1.GetString(file); + var value = text.IndexOf("\nendobj", text.IndexOf("\n9 0 obj\n", StringComparison.Ordinal), StringComparison.Ordinal); + var start = text.LastIndexOf(' ', value) + 1; + return Encoding.Latin1.GetBytes(text[..start] + new string(' ', value - start) + text[value..]); + } + } + + [Theory] + [InlineData("[1 2 3")] + [InlineData("<< /A 1 /B 2")] + [InlineData("<< /A 1 /B")] + [InlineData("<< /A [1 2")] + [InlineData("")] + public void An_object_whose_endobj_ends_its_containers_at_MaxObjectLength_reads_as_under_a_larger_bound(string body) + { + // Object 4 left an array or a dictionary open, or holds nothing, and MaxObjectLength's window ends right after its + // endobj: the file says no regular character follows that endobj, which ends the object as it does under a larger + // bound — the containers it left open reported as the file's —, where it was taken for the guard's cut (#174). Its + // value comes after 100 spaces, so that no other object is as long. + var padded = new string(' ', 100) + body; + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R /Extra 4 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + [4] = file.Object(4, padded), + }; + file.Table(5, rows, "/Size 5 /Root 1 0 R"); + var limit = ("4 0 obj\n" + padded + "\nendobj").Length; + + using var exact = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = limit } }); + using var larger = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = limit + 1 } }); + + exact.GetObject(new PdfObjectId(4)).ToString().Should().Be(larger.GetObject(new PdfObjectId(4)).ToString()); + exact.Reader.IsCutAtLimit(4).Should().BeFalse(); + Of4(exact).Should().Equal(Of4(larger)); + exact.Reader.IsEndObjMissing(4, out _).Should().Be(larger.Reader.IsEndObjMissing(4, out _)); + + var throwing = () => PdfDocument.Open( + file.Bytes, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = limit }, ThrowOnLimit = true }).GetObject(new PdfObjectId(4)); + throwing.Should().NotThrow(); + + // What is reported of object 4, which starts at rows[4]. + IEnumerable<(string, long)> Of4(PdfDocument document) => + document.Diagnostics.Where(entry => entry.Position >= rows[4]).Select(entry => (entry.Code, entry.Position)); } /// From 6a868578ad5c2e130f4501d5341dab22840e2c88 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:50:48 +0000 Subject: [PATCH 066/146] fix: blame no endobj on a reference a window's edge cuts after its generation An object whose value alone is a reference, which MaxObjectLength's window ends inside right after its generation, "6 0 ", the R past the edge, kept the number as an integer and took the generation for what follows the object: the object was said to lack its endobj, at the guard's cut. The batch's chains of /Length and /Prev references write such objects. What follows an integer whose look-ahead the edge cut is now the edge's, as what follows a header it left undecided is. The maintainer had this older defect paid here on 2026-10-23. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 15 ++++++++-- tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs | 30 +++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 86723fef..fc9f5b43 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -76,6 +76,13 @@ internal ref struct PdfObjectParser /// private bool _headerUndecided; + /// + /// Whether a window's edge left the look-ahead of an integer undecided — "12 0" that may be what the edge left of + /// "12 0 R" —: what the parse reads after the integer is what the edge made of the reference, not what follows the + /// object (#224). + /// + private bool _lookAheadCut; + /// /// Whether the last stream read kept a declared length that neither endstream nor endobj follows: where /// its object ends is unknown, and its endobj is not judged. @@ -219,6 +226,7 @@ public PdfObject ParseObject() { _headerAt = -1; _headerUndecided = false; + _lookAheadCut = false; return ParseValue(_lexer.Read(), 0); } @@ -285,6 +293,7 @@ public bool TryReadIndirectObject(out PdfObjectId id, out PdfObject value) _headerStart = number.Start; _headerAt = -1; _headerUndecided = false; + _lookAheadCut = false; // An empty object, "2 0 obj endobj", reads its endobj as its value, and says so; that endobj is its own. var beforeValue = _lexer.Position; @@ -303,8 +312,9 @@ public bool TryReadIndirectObject(out PdfObjectId id, out PdfObject value) var afterValue = _lexer.Position; var next = _lexer.Read(); - // A header a window's edge left undecided ended the value: what follows it is the edge's, not seen (#230). - var unseen = next.Kind == PdfTokenKind.EndOfInput || next.End >= _memory.Length || _headerUndecided; + // A header a window's edge left undecided ended the value, or a reference the edge cut after its generation: what + // follows it is the edge's, not seen (#224, #230). + var unseen = next.Kind == PdfTokenKind.EndOfInput || next.End >= _memory.Length || _headerUndecided || _lookAheadCut; // What follows a stream whose data runs past the buffer lies past it too: the file was asked at the stream's // end, and its answer stands where the buffer has none (#55). @@ -467,6 +477,7 @@ private PdfObject ParseIntegerOrReference(PdfToken token, int depth) if (third.Kind == PdfTokenKind.EndOfInput) { _truncated = true; + _lookAheadCut = !_endsData; } _lexer.Position = saved; diff --git a/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs b/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs index 7ca01538..e831cb43 100644 --- a/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs +++ b/tests/AdCodicem.Pdf.Tests/SyntaxFaultTests.cs @@ -376,6 +376,36 @@ public void An_object_a_guard_cuts_at_a_header_it_leaves_undecided_is_not_said_t new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.ObjectEndObjMissing).Should().BeFalse(); } + [Theory] + [InlineData("6 0 ")] + [InlineData("6 0")] + [InlineData("6 0 R")] + [InlineData("6")] + public void An_object_a_guard_cuts_inside_a_reference_it_holds_alone_is_not_said_to_lack_its_endobj(string kept) + { + // Object 4 holds the reference 6 0 R alone, after 100 spaces, its endobj after it; MaxObjectLength ends the window + // inside it, after its generation, whose look-ahead the edge left undecided: the generation was taken for what follows + // the object, which was said to lack its endobj, at the guard's cut (#224). + var body = new string(' ', 100) + "6 0 R"; + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R /Extra 4 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + [4] = file.Object(4, body), + [6] = file.Object(6, "(six)"), + }; + file.Table(7, rows, "/Size 7 /Root 1 0 R"); + var limit = "4 0 obj\n".Length + 100 + kept.Length; + using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = limit } }); + + document.GetObject(new PdfObjectId(4)); + + document.Diagnostics.Should().Contain(entry => entry.Code == PdfDiagnosticCodes.LimitObject); + document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); + new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.ObjectEndObjMissing).Should().BeFalse(); + } + [Theory] [InlineData("(abc)")] [InlineData("<< /A 1 >>")] From 898bb45a885dc71b43dafcd856f471c696357c2b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:56:39 +0000 Subject: [PATCH 067/146] fix: judge a hybrid stream's own /Prev as written, without following it file.trailer-value-wrong judges a hybrid file's cross-reference stream's own /Prev, which the reader no longer follows, and followed a reference it holds to see whether it leads to a real: the only read of that object, which could reach a guard the reading never met. The value is judged as the file wrote it, a reference or a real where Table 17 asks for a direct integer, and a reference not followed. The maintainer chose this on 2026-10-23. The texts that said no reader follows that /Prev say the reader does not: pdf.js does. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 2 +- .../docs/reference/validation-rules.md | 2 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 5 +++-- .../Validation/Rules/TrailerValueWrongRule.cs | 20 +++++++++++++------ tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 2 +- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 18 +++++++++++++---- 6 files changed, 34 insertions(+), 15 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index 1fbb6105..002c85d6 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -281,7 +281,7 @@ Known gaps: | A missing section tried at many places | Candidates within 512 bytes either side, nearest first, and 32 attempts at most per document (`MaxRelocationCandidates`); a section a guard cut is where it was named, and is tried nowhere else | `CrossReferenceChainTests.Reads_a_section_that_prev_names_a_few_bytes_off`, `Sections_named_a_few_bytes_off_are_looked_for_at_thirty_two_places_a_document_at_most`; `HostileInputTests.Opening_a_chain_of_hybrid_sections_whose_streams_never_close_reads_a_bounded_amount` | Yes | | A classic table of any length | Its window grows ×4 from 64 KB up to `MaxXRefSectionLength`, and the rows read are kept. A table the guard cuts, or whose rows run past a full index, keeps its trailer, found past the rows left unread by a search a megabyte at a time (`ReadTrailerPastRows`): comments are skipped, and anything else no row holds is the section's fault, reported, the search going on to the `trailer` keyword, so that its `/Encrypt` still refuses the document. The search starts at the start of the line the cut fell in, outside any comment and past the table's own keyword; it reads each megabyte once, every byte of it looked at before the next is read, however many comments or stray bytes it holds, and is not made for a place a section may have been moved to, a guess. The chain ends there, since an older section's rows would stand for the objects the unread rows define. What those rows place reads as null until the guard is raised | `ReaderLimitsTests.A_table_whose_keywords_the_bound_cuts_reports_the_bound`, `A_table_its_length_bound_cuts_keeps_its_trailer_and_the_encryption_it_names`, `A_table_its_length_bound_cuts_ends_the_chain_though_its_trailer_names_an_older_section`, `What_ends_the_rows_a_length_bound_cut_is_judged_as_it_is_where_none_did`, `A_table_a_length_bound_cuts_that_runs_to_the_end_of_the_file_has_no_trailer`, `A_table_its_length_bound_cuts_keeps_the_encryption_its_trailer_names_past_what_no_row_holds`, `A_table_cut_before_its_keyword_ends_keeps_its_trailer`, `Rows_past_a_full_index_are_not_read_and_the_trailer_past_them_is`, `A_cut_in_a_comment_among_the_rows_searches_for_the_trailer_from_the_comment_s_start`, `The_trailer_past_rows_left_unread_is_found_wherever_the_search_s_window_ends`, `A_trailer_past_rows_left_unread_that_its_own_bound_cuts_keeps_what_precedes_the_cut`; `DocumentReaderTests.The_search_for_the_trailer_past_rows_left_unread_reads_the_file_once`; `CrossReferenceChainTests.Reads_an_object_a_length_bound_cut_out_of_its_table_as_null_until_the_guard_is_lifted`, `A_section_found_near_where_prev_names_it_is_not_searched_past_the_rows_a_guard_left_unread`; `WindowEdgeTests.A_classic_table_reads_whole_wherever_its_window_edge_falls_near_its_trailer` | No: seeds stay under 120 KB | | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | -| A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_which_no_reader_follows_is_not_read` | Yes | +| A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s` | Yes | | What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain` | No | | A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it, and what decoding it met beside the guard dropped, a filter run over data a spelled `endstream` cut short ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | diff --git a/docs/website/docs/reference/validation-rules.md b/docs/website/docs/reference/validation-rules.md index 18910abc..401cf41f 100644 --- a/docs/website/docs/reference/validation-rules.md +++ b/docs/website/docs/reference/validation-rules.md @@ -45,7 +45,7 @@ References are to ISO 32000-1:2008. | `file.trailer-malformed` | Error | A trailer is not a well-formed dictionary: the `trailer` keyword is not followed by one, or the reader read it despite syntax errors — a key that is not a name, a dictionary or a string left unclosed, a hexadecimal string holding bytes that are neither digits nor white space, a name whose number sign is no escape, a key given twice —, so an entry may have been lost with the syntax, or read otherwise than its writer meant. A cross-reference stream's dictionary is held to the same. | Each trailer of the chain, as the reader parsed it | | `file.root-invalid` | Error | The trailer's `/Root` is missing or does not lead to a document catalog (Table 15), a value that is not a reference included — unless it is a catalog written in the trailer itself, which the reader takes as it is ([#111]). The reader then looks for an object of `/Type /Catalog` among those the index holds — and rebuilds the index only when none is one —: the catalog it finds is its choice, not the file's. Not judged when the trailer could not be read at all, nor when the index was rebuilt as the document opened, nor when one of the reader's limits left the catalog out of the index as the document opened, which `xref.checked-in-part` says, or cut the object `/Root` names — through a chain of references too, and in an object stream whose cut leaves it unread, past the cut data or in a stream the guard made unreadable ([#259]), or in one whose filters end at a `/Filter`, a `/DecodeParms` or a parameter that leads to an object a guard cut or left out of the index, which leaves its data encoded ([#263]) — or the trailer a catalog written there was read from, which the reader then takes for the catalog as it read it, looking for no other, and its `limit.*` report names ([#175]) — whatever a rebuild found since —, nor a `/Root` the trailers lack where one of the reader's limits kept a trailer from being read whole — cut, or older than where the chain stopped, at the section count or at a section whose rows a limit cut or filled the index with —: it may lie there, and the reader rebuilds nothing for it. Rows a limit cut that run on to no trailer lack one: the file's, which a rebuild answers. The catalog the finding names is the one the reader took last, in the rebuild run before the first rule if there was one. | The trailer the chain gave, merged as readers merge it, and `/Root` before the reader looked for a catalog | | `file.size-wrong` | Warning | A section's `/Size` is missing, or is not one more than the highest object number it and the sections it updates use (Tables 15 and 17, 7.5.6, 7.5.8.4). A hybrid file's cross-reference stream meets one of Table 17's two readings or the other — one more than the highest number it and the sections it updates use, or the `/Size` of the table that names it —, and either is accepted. A `/Size` that leaves in-use objects numbered above it is `xref.object-past-size`'s. A `/Size` written as a reference, or as a real, is said to be. Not judged when the chain lost a section, or one of the reader's limits cut it or ended it before its older sections ([#263]). | Each section's trailer, and the highest number each section indexes | -| `file.trailer-value-wrong` | Warning | A value a section's trailer gives the reader to read the chain is written as the specification does not allow, and read all the same, as other readers read it: a reference where it must be direct, read where a section read before it places the object; a real with no fractional part where it must be an integer, read as the integer it equals ([#182], [#215]). A cross-reference stream's `/Type`, `/Index`, `/Prev` and `/W`, the elements of `/Index` and `/W`, and its `/Filter` and `/DecodeParms` are direct (7.5.8.2); in a file that declares PDF 2.0, so are its `/Length` and the elements and entries of its `/Filter` and `/DecodeParms` (an erratum of ISO 32000-2, pdf-issues #246). A classic trailer's `/Prev` is direct (Table 15); an `/XRefStm` may be indirect (Table 19). `/W`, `/Index`, `/Prev`, `/XRefStm`, a stream's `/Length` and its `/Predictor`, `/Colors`, `/BitsPerComponent`, `/Columns` and `/EarlyChange` are integers. One finding per value, which says both faults of a reference that leads to a real. A value the chain is read with that the reader could not read so is the section's fault — `xref.section-malformed`, `xref.section-not-found` —, unless it leads to an object one of the reader's limits cut, where the chain ends for the limit, a section whose rows need it left to `xref.checked-in-part` ([#263]), and `/Size` is `file.size-wrong`'s; a `/Type`, or a `/Length` read once the chain is, is reported here whether or not it could be read. | Each section's trailer, as the file wrote it, and the objects its references name, through the index the reader settled on before the first rule | +| `file.trailer-value-wrong` | Warning | A value a section's trailer gives the reader to read the chain is written as the specification does not allow, and read all the same, as other readers read it: a reference where it must be direct, read where a section read before it places the object; a real with no fractional part where it must be an integer, read as the integer it equals ([#182], [#215]). A cross-reference stream's `/Type`, `/Index`, `/Prev` and `/W`, the elements of `/Index` and `/W`, and its `/Filter` and `/DecodeParms` are direct (7.5.8.2); in a file that declares PDF 2.0, so are its `/Length` and the elements and entries of its `/Filter` and `/DecodeParms` (an erratum of ISO 32000-2, pdf-issues #246). A classic trailer's `/Prev` is direct (Table 15); an `/XRefStm` may be indirect (Table 19). A hybrid file's cross-reference stream's own `/Prev`, which the reader does not follow, the chain going on through the table's, is judged as the file wrote it, a reference it holds not followed. `/W`, `/Index`, `/Prev`, `/XRefStm`, a stream's `/Length` and its `/Predictor`, `/Colors`, `/BitsPerComponent`, `/Columns` and `/EarlyChange` are integers. One finding per value, which says both faults of a reference that leads to a real. A value the chain is read with that the reader could not read so is the section's fault — `xref.section-malformed`, `xref.section-not-found` —, unless it leads to an object one of the reader's limits cut, where the chain ends for the limit, a section whose rows need it left to `xref.checked-in-part` ([#263]), and `/Size` is `file.size-wrong`'s; a `/Type`, or a `/Length` read once the chain is, is reported here whether or not it could be read. | Each section's trailer, as the file wrote it, and the objects its references name, through the index the reader settled on before the first rule | ### Cross-references diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 449d7713..dfac054b 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3868,8 +3868,9 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect _xref.MergeTrailer(dictionary); - // A hybrid file's stream gives a /Prev of its own, which no reader follows: the chain goes on through the table's, - // and the stream's is not read, to name a section missing or end the chain for a guard. + // A hybrid file's stream gives a /Prev of its own, which the reader does not follow: the chain goes on through the + // table's, as ISO 32000-1 (7.5.8.4) describes, and the stream's is not read, to name a section missing or end the + // chain for a guard. string? previousHidden = null; previous = rowsCut || section.NamedBy == "/XRefStm" ? -1 : SectionOffset(dictionary, PdfName.Prev, absolute, out previousHidden); section.ChainHiddenByGuard = previousHidden; diff --git a/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs b/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs index c63832cb..8fdbf406 100644 --- a/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs +++ b/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs @@ -30,7 +30,9 @@ namespace AdCodicem.Pdf.Validation.Rules; /// /// Each trailer is judged as the file wrote it; a reference it holds is followed as any read follows it, through the index /// the validator had the reader settle on before its first rule — the chain's own, unless a read could still have rebuilt -/// it, in which case it was rebuilt then (#237) —, so that what was read before validating changes nothing. +/// it, in which case it was rebuilt then (#237) —, so that what was read before validating changes nothing. A hybrid file's +/// stream's own /Prev, which the reader does not follow, the chain going on through the table's, is judged as the +/// file wrote it, a reference it holds not followed (#263). /// /// internal sealed class TrailerValueWrongRule : IValidationRule @@ -72,9 +74,11 @@ public void Check(ValidationContext context) judge.Integers(trailer, PdfName.W); judge.Integers(trailer, PdfName.Index); + // A hybrid file's stream gives a /Prev of its own, which the reader does not follow, the chain going on through + // the table's (7.5.8.4): judged as the file wrote it, a reference it holds not followed (#263). if (IsFollowed(sections, section, PdfName.Prev)) { - judge.Value("its /Prev", trailer.GetRaw(PdfName.Prev), Iso1, "Table 17"); + judge.Value("its /Prev", trailer.GetRaw(PdfName.Prev), Iso1, "Table 17", resolve: section.NamedBy != "/XRefStm"); } judge.Value("its /Length", trailer.GetRaw(PdfName.Length), pdf20 ? Iso2 : null, "Table 5"); @@ -96,8 +100,8 @@ public void Check(ValidationContext context) } /// - /// Determines whether the chain went on through the of : one that - /// names no section is 's to report. + /// Determines whether the of named a section the chain found, or none + /// it looked for: one that names no section is 's to report. /// private static bool IsFollowed(IReadOnlyList sections, XRefSectionRecord section, PdfName key) { @@ -130,10 +134,14 @@ private readonly struct Judge(ValidationContext context, IValidationRule rule, X /// The value as the file wrote it, or null when there is none. /// What makes the value direct, or null when nothing does. /// The table that makes the value an integer, or null when none does. - public void Value(string what, PdfObject? value, string? direct, string? integer) + /// + /// Whether a reference the value is written as is followed to judge what it leads to, which a value the reader does + /// not read is not: following it would read what the reading never did. + /// + public void Value(string what, PdfObject? value, string? direct, string? integer, bool resolve = true) { var reference = direct is not null ? value as PdfReference : null; - var real = integer is not null && value?.Resolve() is PdfReal written && written.AsInteger() is not null; + var real = integer is not null && (resolve ? value?.Resolve() : value) is PdfReal written && written.AsInteger() is not null; if (reference is null && !real) { diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 113688db..86de386e 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -589,7 +589,7 @@ public static byte[] UpdatePlacingAPaddedLengthChain(int objects, bool ownRow) /// /// A hybrid update over a classic table that indexes the catalog, the page tree and the page. The update writes object 9, /// which holds 123 after spaces, and object 10, which cross-reference stream 7 indexes; the - /// stream's own /Prev is , which no reader follows. The update's table indexes object + /// stream's own /Prev is , which the reader does not follow. The update's table indexes object /// 9, and its trailer names the older table through /Prev and the stream through /XRefStm. /// public static byte[] HybridWithStreamPrev(string streamPrev, int padding = 3000) diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index bea5beee..a5e2cef0 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -7,6 +7,7 @@ using AdCodicem.Pdf.IO.XRef; using AdCodicem.Pdf.Objects; using AdCodicem.Pdf.Validation; +using AdCodicem.Pdf.Validation.Rules; namespace AdCodicem.Pdf.Tests; @@ -995,11 +996,14 @@ public void A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_gu [InlineData("9 0 R", true)] [InlineData("9 0 R", false)] [InlineData("12 0 R", false)] - public void A_hybrid_stream_s_own_prev_which_no_reader_follows_is_not_read(string prev, bool guarded) + [InlineData("4.0", false)] + public void A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s(string prev, bool guarded) { - // A hybrid file's stream gives a /Prev of its own, which the chain does not follow: it goes on through the table's. - // Read anyway, one that leads to an object MaxObjectLength cuts ended the chain for the guard, the older table lost, - // and one no section places was said to name a missing section (#263). + // A hybrid file's stream gives a /Prev of its own, which the chain does not follow: it goes on through the table's, + // as ISO 32000-1 (7.5.8.4) describes. Read anyway, one that leads to an object MaxObjectLength cuts ended the chain + // for the guard, the older table lost, and one no section places was said to name a missing section (#263). The + // rule on trailer values judges it as the file wrote it, a reference or a real where Table 17 asks for a direct + // integer, without following it: following it reached the guard the reading never met. var file = ChainFiles.HybridWithStreamPrev(prev); using var document = PdfDocument.Open(file, guarded ? ObjectGuard : null); @@ -1010,6 +1014,12 @@ public void A_hybrid_stream_s_own_prev_which_no_reader_follows_is_not_read(strin var report = new PdfValidator().Validate(document); report.Contains(PdfValidationRuleIds.XRefSectionNotFound).Should().BeFalse(); report.Contains(PdfValidationRuleIds.XRefCheckedInPart).Should().BeFalse(); + + using var judged = PdfDocument.Open(file, guarded ? ObjectGuard : null); + var trailers = new ValidationProfile("trailers", 1, [new TrailerValueWrongRule()]); + new PdfValidator(new PdfValidatorOptions { Profile = trailers }).Validate(judged).Findings.Select(finding => finding.Message) + .Should().ContainSingle().Which.Should().Contain(prev == "4.0" ? "gives its /Prev as a real number" : $"writes its /Prev as the reference {prev}"); + judged.Diagnostics.Should().BeEmpty(); } [Theory] From 0ecf0e40f6013ff7974309c42c63249faaa75668 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:57:44 +0000 Subject: [PATCH 068/146] docs: say what a hidden /XRefStm names and which parsers ask the file's edge The pages on file.size-wrong, page-tree.page-orphaned, lazy reading and the reader's limits said a /Prev or /XRefStm a guard hid leaves "older sections" unread: an /XRefStm names the cross-reference stream of a hybrid file's revision, which they now say. IPdfFileEdge's remarks said only a parser at MaxObjectLength's last window asks the file of an endobj its edge reaches: any parser whose window does not end the data does. The threat model cites the tests the fourth review round added, and says an endobj that ends a container or an empty object at the bound, and a reference the edge cuts after its generation, are the edge's. Refs #174, #224, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 4 ++-- docs/website/docs/concepts/lazy-reading.md | 3 ++- docs/website/docs/reference/reader-limits.md | 4 ++-- docs/website/docs/reference/validation-rules.md | 4 ++-- src/AdCodicem.Pdf/IO/IPdfFileEdge.cs | 8 ++++---- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index 002c85d6..93b8ab78 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -248,7 +248,7 @@ Known gaps: | The lexer looping, or reading past its buffer | Every token but the end of input consumes a byte. Look-aheads check the length, and a string that never closes ends at the buffer's end, and is told from a closed one by its token's length | `PropertyTests.The_lexer_terminates_on_any_bytes_and_stays_inside_the_buffer`; `LexerTests.Never_loops_on_a_stray_delimiter`, `Recovers_from_an_unterminated_literal_string`; `ParserTests.Reports_nothing_of_a_construct_closed_at_the_end_of_the_data` | Property, over buffers of about 100 bytes | | Nesting that exhausts the stack | `PdfObjectParser.MaxDepth` (128) and a stack probe at each container. Past either, `SkipContainer` counts brackets without recursion, and `syntax.depth-exceeded` is reported | `ParserTests.Refuses_to_follow_nesting_deeper_than_its_limit`; `HostileInputTests.Survives_deeply_nested_containers_inside_an_object`, `A_file_nesting_objects_deeper_than_the_stack_allows_reads_them_as_null_without_crashing` | No | | A token or an object of any length, or one that never closes | The window grows ×8 from 8 KB up to `MaxObjectLength`, and the parser is told whether its window ends the data: only then is what it leaves open reported, once, `syntax.truncated-object` where the innermost construct opens, and a window that ends where the file does cuts nothing however long it is. Past the bound, the object is kept as far as it was read and `limit.object` is reported, beside what the parse met before the cut; nothing is reported of the token the cut reaches, nor of what follows once a look-ahead met it (ADR 34). An array or a dictionary an `endobj` finds open ends there, rather than take the objects after it, and one that lost its own `endobj` too ends at the next object's header (below). The parses of one object add up to about 1.3 times the last window | `HostileInputTests.Survives_a_string_longer_than_the_largest_window`, `A_string_that_never_closes_takes_the_end_of_the_file_and_is_reported_once_where_it_opens`; `ReaderLimitsTests.An_object_is_read_no_further_than_its_bound`, `An_object_cut_by_its_bound_is_not_reported_as_damage`, `An_object_its_bound_cuts_anywhere_in_its_last_tokens_reports_the_bound_alone`, `What_an_object_its_bound_cuts_met_before_the_cut_is_still_reported`, `A_fault_after_a_stream_whose_length_the_file_cannot_hold_is_still_reported_beside_the_bound`, `An_object_its_bound_cuts_after_more_faults_than_the_diagnostics_hold_still_reports_the_bound`, `A_number_its_bound_cuts_is_no_number_out_of_range`, `An_object_as_long_as_its_bound_that_ends_the_file_is_read_whole`; `WindowEdgeTests.An_anomaly_inside_an_object_longer_than_the_window_is_reported_once`; `PropertyTests.What_a_window_reports_of_its_syntax_the_whole_data_reports_too_in_the_same_order`, `A_buffer_that_ends_at_a_window_s_edge_reports_nothing_of_the_end_of_the_data`; `SyntaxFaultTests.A_construct_the_file_ends_inside_is_reported_where_it_opens`, `A_container_an_endobj_ends_takes_nothing_of_the_objects_after_it`, `A_fault_of_an_object_read_through_a_window_the_file_goes_on_past_is_reported`, `A_rebuild_reports_no_fault_where_its_window_cuts_a_trailer`, `A_trailer_a_source_cut_short_ends_inside_is_reported_as_the_end_of_the_data`; `FileRuleTests.A_trailer_whose_closing_brackets_end_its_table_s_window_is_judged_by_what_they_close` | Growth yes, the bound no; the window's edge by property | -| An object that lost both its closing delimiter and its `endobj` taking in the next object, up to that object's `endobj`, its own missing `endobj` unreported; a trailer that lost its `>>` taking the next update's entries; a header the window's edge cut short reported as the file's stray keys | Inside an array or a dictionary, where an element, a key or a value is expected, the next object's header ends every container left open: `N G obj`, digits alone, `N` in 1..`int.MaxValue` and `G` in 0..65,535, white space alone between them, and `obj` a whole keyword the window's edge does not reach — the header the search for a lost `endstream` stops at (`HeaderText`). A header the edge cuts short, `6 0` or `6 0 ob`, is the edge's, as a reference it cuts is: where no larger window follows — at `MaxObjectLength`, in the rebuild's trailer scan —, its numbers are not reported as keys that are not names, unless a whole token already rules a header out — a number 0, a generation past 65,535, a member, which no header follows —; `6 0 ob`, a keyword `obj` starts after two integers, which no reference can be, ends the containers as a header does, its number given to no key and nothing reported, while `6 0` alone, what the edge may have left of a reference, is kept as far as it was read, as any value the edge cuts. It is reported once, at the innermost container, as `syntax.truncated-object`, "An object header ended the object inside …", or, in a trailer or a value read alone, where no object is open, "An object header came inside …", and the object lacks its `endobj`. Containers `SkipContainer` passes over past `MaxDepth` end there too, and a trailer is read so as well. An object stream's member, which no header follows, and a value alone, which is no container, do not apply it: in `4 0 obj 5 0 obj`, object 4 holds the integer 5. Its cost, accepted: a closed container holding text that reads as a header ends there, and its `endobj` is judged missing; none of the 344,361 objects of the corpus holds one ([#230]). An `endobj` the window of `MaxObjectLength` ends exactly with is asked of the file, which says whether a regular character goes on past it: an object exactly as long as the guard reads whole, loaded or read by the chain ([#174]) | `SyntaxFaultTests.An_object_that_lost_its_closing_delimiter_and_its_endobj_ends_at_the_next_object_s_header`, `The_next_object_s_header_ends_every_container_left_open_and_is_reported_at_the_innermost`, `Text_that_is_no_object_header_ends_no_container`, `A_header_inside_a_closed_container_ends_it_there_and_its_endobj_is_judged_missing`, `An_object_stream_member_holding_a_header_reads_it_as_tokens`, `Nesting_deeper_than_the_reader_follows_ends_at_the_next_object_s_header_too`, `A_trailer_that_lost_its_closing_delimiter_takes_nothing_of_the_update_after_it`, `A_header_the_guard_s_cut_leaves_undecided_is_the_guard_s`; `ParserTests.Ends_the_containers_an_object_left_open_at_the_next_object_s_header_and_leaves_it_to_be_read`, `Takes_an_object_header_a_window_s_edge_reaches_for_none_until_a_larger_window_shows_obj_whole`, `A_header_where_a_key_or_a_value_is_expected_ends_the_dictionary`, `A_value_alone_that_is_a_header_s_number_is_an_integer`, `Tokens_a_window_s_edge_leaves_undecided_as_a_header_are_no_stray_keys`; `WindowEdgeTests.An_object_that_lost_its_end_ends_at_the_next_header_wherever_the_window_edge_cuts_it`; `SyntaxFaultTests.A_header_the_guard_s_cut_leaves_undecided_gives_the_key_before_it_no_value`, `A_value_no_header_can_begin_with_keeps_its_place_at_the_guard_s_cut`, `An_object_a_guard_cuts_at_a_header_it_leaves_undecided_is_not_said_to_lack_its_endobj`, `Tokens_no_header_begins_with_are_reported_at_the_guard_s_cut_as_past_it`, `An_object_exactly_as_long_as_MaxObjectLength_reads_whole`, `An_object_the_chain_reads_exactly_as_long_as_MaxObjectLength_reads_whole`; `ParserTests.Ends_a_container_skipped_for_its_depth_at_the_next_object_s_header`, `Reports_the_keys_of_a_member_a_window_s_edge_leaves_as_a_header_none_can_be` | Plausibly | +| An object that lost both its closing delimiter and its `endobj` taking in the next object, up to that object's `endobj`, its own missing `endobj` unreported; a trailer that lost its `>>` taking the next update's entries; a header the window's edge cut short reported as the file's stray keys | Inside an array or a dictionary, where an element, a key or a value is expected, the next object's header ends every container left open: `N G obj`, digits alone, `N` in 1..`int.MaxValue` and `G` in 0..65,535, white space alone between them, and `obj` a whole keyword the window's edge does not reach — the header the search for a lost `endstream` stops at (`HeaderText`). A header the edge cuts short, `6 0` or `6 0 ob`, is the edge's, as a reference it cuts is: where no larger window follows — at `MaxObjectLength`, in the rebuild's trailer scan —, its numbers are not reported as keys that are not names, unless a whole token already rules a header out — a number 0, a generation past 65,535, a member, which no header follows —; `6 0 ob`, a keyword `obj` starts after two integers, which no reference can be, ends the containers as a header does, its number given to no key and nothing reported, while `6 0` alone, what the edge may have left of a reference, is kept as far as it was read, as any value the edge cuts. It is reported once, at the innermost container, as `syntax.truncated-object`, "An object header ended the object inside …", or, in a trailer or a value read alone, where no object is open, "An object header came inside …", and the object lacks its `endobj`. Containers `SkipContainer` passes over past `MaxDepth` end there too, and a trailer is read so as well. An object stream's member, which no header follows, and a value alone, which is no container, do not apply it: in `4 0 obj 5 0 obj`, object 4 holds the integer 5. Its cost, accepted: a closed container holding text that reads as a header ends there, and its `endobj` is judged missing; none of the 344,361 objects of the corpus holds one ([#230]). An `endobj` the window of `MaxObjectLength` ends exactly with is asked of the file, which says whether a regular character goes on past it: an object exactly as long as the guard reads whole, loaded or read by the chain, wherever its `endobj` comes — after its value, where a container it left open is, or as all an empty object holds ([#174]). A reference alone that the window's edge cuts after its generation leaves what follows it the edge's: the object is not said to lack its `endobj` at the guard's cut ([#224]) | `SyntaxFaultTests.An_object_that_lost_its_closing_delimiter_and_its_endobj_ends_at_the_next_object_s_header`, `The_next_object_s_header_ends_every_container_left_open_and_is_reported_at_the_innermost`, `Text_that_is_no_object_header_ends_no_container`, `A_header_inside_a_closed_container_ends_it_there_and_its_endobj_is_judged_missing`, `An_object_stream_member_holding_a_header_reads_it_as_tokens`, `Nesting_deeper_than_the_reader_follows_ends_at_the_next_object_s_header_too`, `A_trailer_that_lost_its_closing_delimiter_takes_nothing_of_the_update_after_it`, `A_header_the_guard_s_cut_leaves_undecided_is_the_guard_s`; `ParserTests.Ends_the_containers_an_object_left_open_at_the_next_object_s_header_and_leaves_it_to_be_read`, `Takes_an_object_header_a_window_s_edge_reaches_for_none_until_a_larger_window_shows_obj_whole`, `A_header_where_a_key_or_a_value_is_expected_ends_the_dictionary`, `A_value_alone_that_is_a_header_s_number_is_an_integer`, `Tokens_a_window_s_edge_leaves_undecided_as_a_header_are_no_stray_keys`; `WindowEdgeTests.An_object_that_lost_its_end_ends_at_the_next_header_wherever_the_window_edge_cuts_it`; `SyntaxFaultTests.A_header_the_guard_s_cut_leaves_undecided_gives_the_key_before_it_no_value`, `A_value_no_header_can_begin_with_keeps_its_place_at_the_guard_s_cut`, `An_object_a_guard_cuts_at_a_header_it_leaves_undecided_is_not_said_to_lack_its_endobj`, `Tokens_no_header_begins_with_are_reported_at_the_guard_s_cut_as_past_it`, `An_object_exactly_as_long_as_MaxObjectLength_reads_whole`, `An_object_the_chain_reads_exactly_as_long_as_MaxObjectLength_reads_whole`, `An_object_whose_endobj_ends_its_containers_at_MaxObjectLength_reads_as_under_a_larger_bound`, `An_object_a_guard_cuts_inside_a_reference_it_holds_alone_is_not_said_to_lack_its_endobj`; `ParserTests.Ends_a_container_skipped_for_its_depth_at_the_next_object_s_header`, `Reports_the_keys_of_a_member_a_window_s_edge_leaves_as_a_header_none_can_be` | Plausibly | | Look-ahead that makes parsing superlinear | Look-ahead is a constant number of tokens, so each token is lexed three times at most | `ParserTests.Distinguishes_a_reference_from_two_integers`, `Backtracks_when_two_integers_are_not_followed_by_R` | Property | | A number past a `long` or a `double` | The number parser never wraps: an integer past a `long` reads as the real nearest to it, and a number past a double's range as an infinity, which the object parser reads as null under `syntax.number-out-of-range`, quoting the number only in a report the diagnostics keep. A real reads as the double nearest to its decimal. The shape — signs, then digits around at most one period — is checked before the framework's parse, which would read `NaN` and `Infinity`; each digit is read twice at most, and nothing is allocated | `LexerTests.Reads_an_integer_too_large_for_a_long_as_a_real`, `Reads_a_number_past_the_largest_double_as_an_infinity_its_reader_can_tell`, `Parsing_a_number_allocates_nothing_however_long_it_is`; `PropertyTests.The_integer_parser_reads_every_long_and_never_wraps_past_one`, `The_real_parser_reads_the_double_nearest_to_the_decimal`; `ParserTests.Reads_a_number_past_the_largest_real_as_null_and_reports_it`, `Reports_no_number_its_double_holds_however_it_rounds`; `HostileInputTests.Reads_a_number_fifteen_million_digits_long_in_time`, `Quotes_no_number_beyond_what_a_real_can_hold_once_the_diagnostics_are_full` | Property | | An object number or a generation overflowing its cast | The parser builds a header only from a number in 1..`int.MaxValue`, and a reference only from one in 0..`int.MaxValue` — object 0 heads the free list and reads as null —, each with a generation in 0..65,535 (`PdfObjectId.MaxNumber` and `MaxGeneration`, with their reasons), checked before the cast. The index and object streams check theirs the same way, before narrowing: see *Indexing* and *Resolving objects and object streams* | `ParserTests.Refuses_an_object_header_whose_number_or_generation_is_out_of_range`, `Makes_no_reference_of_numbers_an_object_identifier_cannot_hold`; `HostileInputTests.Serves_no_object_under_a_number_past_a_long_that_once_wrapped_to_its_number` | No | @@ -283,7 +283,7 @@ Known gaps: | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s` | Yes | | What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it, and what decoding it met beside the guard dropped, a filter run over data a spelled `endstream` cut short ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it, and what decoding it met beside the guard dropped, a filter run over data a spelled `endstream` cut short ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | diff --git a/docs/website/docs/concepts/lazy-reading.md b/docs/website/docs/concepts/lazy-reading.md index d9030a9e..eb8667ee 100644 --- a/docs/website/docs/concepts/lazy-reading.md +++ b/docs/website/docs/concepts/lazy-reading.md @@ -62,7 +62,8 @@ sections past `MaxXRefSectionCount`, the rows past `MaxXRefSectionLength` or pas `/Filter` or `/DecodeParms` leads to an object `MaxObjectLength` cuts and the sections older than it — so too one whose `/Length` leads to such an object, when no `endstream` ends its data before the next object or the end of the file, or the rows past an `endstream` its rows spell, which only that length could tell from the end of its data —, the sections -older than a `/Prev` or an `/XRefStm` that leads to such an object, the numbers past `MaxIndexEntries`. That is no +a `/Prev` or an `/XRefStm` that leads to such an object names — the older ones, or the cross-reference stream of a hybrid +file's revision —, the numbers past `MaxIndexEntries`. That is no damage, and the map is not rebuilt for it. The map is rebuilt by scanning the whole file, once at most, and never while the chain is read: diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index d745b988..285c2971 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -121,8 +121,8 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. is among them the reader does not look for another. While the chain is read, a `/Prev` or an `/XRefStm` that leads to an object `MaxObjectLength` cuts, a cross-reference stream whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` does, and one whose `/Length` does when no `endstream` ends its data, are the guard's, as a section it cut is: the - chain ends there, the older sections are not read, the index is not rebuilt, and the guard is reported once the chain - is read, or thrown by `Open`. + chain ends there, the sections it names are not read — the older ones, or a hybrid file's cross-reference stream —, + the index is not rebuilt, and the guard is reported once the chain is read, or thrown by `Open`. - A stream read from a document decodes under **that document's** limits, however long after opening you decode it. A stream you build in memory decodes under `PdfReaderLimits.Default`. - A classic trailer that ends within the window its cross-reference table was read through is read whole, whatever diff --git a/docs/website/docs/reference/validation-rules.md b/docs/website/docs/reference/validation-rules.md index 401cf41f..131c8db9 100644 --- a/docs/website/docs/reference/validation-rules.md +++ b/docs/website/docs/reference/validation-rules.md @@ -44,7 +44,7 @@ References are to ISO 32000-1:2008. | `file.trailer-missing` | Error | A cross-reference table's rows are not followed by the `trailer` keyword: they run into a dictionary, or the file ends. qpdf, as the reader, then rebuilds the index. | Each classic table of the chain | | `file.trailer-malformed` | Error | A trailer is not a well-formed dictionary: the `trailer` keyword is not followed by one, or the reader read it despite syntax errors — a key that is not a name, a dictionary or a string left unclosed, a hexadecimal string holding bytes that are neither digits nor white space, a name whose number sign is no escape, a key given twice —, so an entry may have been lost with the syntax, or read otherwise than its writer meant. A cross-reference stream's dictionary is held to the same. | Each trailer of the chain, as the reader parsed it | | `file.root-invalid` | Error | The trailer's `/Root` is missing or does not lead to a document catalog (Table 15), a value that is not a reference included — unless it is a catalog written in the trailer itself, which the reader takes as it is ([#111]). The reader then looks for an object of `/Type /Catalog` among those the index holds — and rebuilds the index only when none is one —: the catalog it finds is its choice, not the file's. Not judged when the trailer could not be read at all, nor when the index was rebuilt as the document opened, nor when one of the reader's limits left the catalog out of the index as the document opened, which `xref.checked-in-part` says, or cut the object `/Root` names — through a chain of references too, and in an object stream whose cut leaves it unread, past the cut data or in a stream the guard made unreadable ([#259]), or in one whose filters end at a `/Filter`, a `/DecodeParms` or a parameter that leads to an object a guard cut or left out of the index, which leaves its data encoded ([#263]) — or the trailer a catalog written there was read from, which the reader then takes for the catalog as it read it, looking for no other, and its `limit.*` report names ([#175]) — whatever a rebuild found since —, nor a `/Root` the trailers lack where one of the reader's limits kept a trailer from being read whole — cut, or older than where the chain stopped, at the section count or at a section whose rows a limit cut or filled the index with —: it may lie there, and the reader rebuilds nothing for it. Rows a limit cut that run on to no trailer lack one: the file's, which a rebuild answers. The catalog the finding names is the one the reader took last, in the rebuild run before the first rule if there was one. | The trailer the chain gave, merged as readers merge it, and `/Root` before the reader looked for a catalog | -| `file.size-wrong` | Warning | A section's `/Size` is missing, or is not one more than the highest object number it and the sections it updates use (Tables 15 and 17, 7.5.6, 7.5.8.4). A hybrid file's cross-reference stream meets one of Table 17's two readings or the other — one more than the highest number it and the sections it updates use, or the `/Size` of the table that names it —, and either is accepted. A `/Size` that leaves in-use objects numbered above it is `xref.object-past-size`'s. A `/Size` written as a reference, or as a real, is said to be. Not judged when the chain lost a section, or one of the reader's limits cut it or ended it before its older sections ([#263]). | Each section's trailer, and the highest number each section indexes | +| `file.size-wrong` | Warning | A section's `/Size` is missing, or is not one more than the highest object number it and the sections it updates use (Tables 15 and 17, 7.5.6, 7.5.8.4). A hybrid file's cross-reference stream meets one of Table 17's two readings or the other — one more than the highest number it and the sections it updates use, or the `/Size` of the table that names it —, and either is accepted. A `/Size` that leaves in-use objects numbered above it is `xref.object-past-size`'s. A `/Size` written as a reference, or as a real, is said to be. Not judged when the chain lost a section, or one of the reader's limits cut it or ended it before the sections a `/Prev` or an `/XRefStm` names ([#263]). | Each section's trailer, and the highest number each section indexes | | `file.trailer-value-wrong` | Warning | A value a section's trailer gives the reader to read the chain is written as the specification does not allow, and read all the same, as other readers read it: a reference where it must be direct, read where a section read before it places the object; a real with no fractional part where it must be an integer, read as the integer it equals ([#182], [#215]). A cross-reference stream's `/Type`, `/Index`, `/Prev` and `/W`, the elements of `/Index` and `/W`, and its `/Filter` and `/DecodeParms` are direct (7.5.8.2); in a file that declares PDF 2.0, so are its `/Length` and the elements and entries of its `/Filter` and `/DecodeParms` (an erratum of ISO 32000-2, pdf-issues #246). A classic trailer's `/Prev` is direct (Table 15); an `/XRefStm` may be indirect (Table 19). A hybrid file's cross-reference stream's own `/Prev`, which the reader does not follow, the chain going on through the table's, is judged as the file wrote it, a reference it holds not followed. `/W`, `/Index`, `/Prev`, `/XRefStm`, a stream's `/Length` and its `/Predictor`, `/Colors`, `/BitsPerComponent`, `/Columns` and `/EarlyChange` are integers. One finding per value, which says both faults of a reference that leads to a real. A value the chain is read with that the reader could not read so is the section's fault — `xref.section-malformed`, `xref.section-not-found` —, unless it leads to an object one of the reader's limits cut, where the chain ends for the limit, a section whose rows need it left to `xref.checked-in-part` ([#263]), and `/Size` is `file.size-wrong`'s; a `/Type`, or a `/Length` read once the chain is, is reported here whether or not it could be read. | Each section's trailer, as the file wrote it, and the objects its references name, through the index the reader settled on before the first rule | ### Cross-references @@ -110,7 +110,7 @@ object nothing reachable refers to is not judged by them. | `page-tree.parent-wrong` | Warning | A node or a page has no `/Parent`, or one that is not the node listing it; or the root has one (Tables 29 and 30). The reader, as qpdf, walks the tree through `/Kids` and inherits along that path. | The page tree | | `page-tree.mediabox-invalid` | Warning | A page has no `/MediaBox`, its own or inherited, or one that is not four numbers enclosing an area (Table 30, 7.9.5): its size is unknown, and qpdf gives it a US Letter sheet's. Any two opposite corners make a rectangle, so a box written from its upper corner is sound. Missing, reported for each page; malformed, where it is written. | The page tree, and each media box | | `page-tree.resources-missing` | Warning | A page has no `/Resources`, its own or inherited (Table 30), where an empty dictionary says it uses none. | The page tree | -| `page-tree.page-orphaned` | Information | An object of `/Type /Page`, with a `/Parent` or a `/Contents`, that the page tree does not list: no reader shows it. Nothing forbids one — an edit leaves them —; a marked-content property list typed `/Page`, as DocuSign writes, is not a page. Judged only where the file's own index is sound and whole — no section lost, cut, or whose older sections one of the reader's limits hid ([#263]) — and the tree was read whole — no part of it left unread or cut by one of the reader's limits —, since elsewhere loading an object may bring back what an update deleted, and a page the tree seems not to list may lie past a cut. | Every object the file's index holds in use, once each, without its content | +| `page-tree.page-orphaned` | Information | An object of `/Type /Page`, with a `/Parent` or a `/Contents`, that the page tree does not list: no reader shows it. Nothing forbids one — an edit leaves them —; a marked-content property list typed `/Page`, as DocuSign writes, is not a page. Judged only where the file's own index is sound and whole — no section lost, cut, or whose `/Prev` or `/XRefStm` one of the reader's limits hid, with the sections it names ([#263]) — and the tree was read whole — no part of it left unread or cut by one of the reader's limits —, since elsewhere loading an object may bring back what an update deleted, and a page the tree seems not to list may lie past a cut. | Every object the file's index holds in use, once each, without its content | The page tree is walked once from the catalog's `/Pages`, through `/Kids`, as qpdf walks it: a node is a dictionary with a `/Kids` array or of `/Type /Pages`, any other dictionary the tree lists is a page, and pages are counted as the diff --git a/src/AdCodicem.Pdf/IO/IPdfFileEdge.cs b/src/AdCodicem.Pdf/IO/IPdfFileEdge.cs index 87feba7a..17758948 100644 --- a/src/AdCodicem.Pdf/IO/IPdfFileEdge.cs +++ b/src/AdCodicem.Pdf/IO/IPdfFileEdge.cs @@ -4,10 +4,10 @@ namespace AdCodicem.Pdf.IO; /// Says what the file holds right past the end of a parser's buffer, where a window's edge leaves a token undecided. /// /// -/// A parser given no larger window — one ends — asks it of a -/// keyword its window's edge reaches, which may be the start of a longer one: an object a window ends exactly with reads -/// whole, as under a larger window (#174). It is all the parser asks of the file while the cross-reference chain is read, -/// which reads no stream's data. +/// A parser whose window does not end the data asks it of an endobj its window's edge reaches, which may be the start +/// of a longer keyword: an object a window ends exactly with — the last one +/// allows, say — reads whole, as under a larger window (#174). It is +/// all a parser asks of the file when it reads an object for the chain without loading it, which reads no stream's data. /// internal interface IPdfFileEdge { From a6747945716ec66121870c8c037b808ef7753e71 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 19:14:34 +0000 Subject: [PATCH 069/146] fix: read through a whole reference a guard cut the object after, as a load does The chain read an object MaxObjectLength cut as nothing, even when the cut fell after a whole reference the object held. A value the chain needs ended there for the guard: a /W or an /Index left the section unread, and a /Length left the data at the first endstream its rows spell. Yet the guard hid nothing the value needs, and a load reads through such an object to the one it names, whose own state decides. The chain now reads through it too, and loads the object once the chain is read, so that its guard is reached, reported, or thrown under ThrowOnLimit. What the chain kept of a cut object it had judged when it first read it: a section placing the named object later found it unreadable, and the data a newer section's /Length gave could not be taken again. Only an integer the cut left is kept now. A length reached that way is the file's. Rows the data taken again to it still lacks are the file's to lack, as when read whole; only an integer a cut left is the guard's. A test holds the own row's object that a guard parts, which no test reached. Refs #224, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 111 ++++++++++++------ tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 99 ++++++++++++++++ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 111 ++++++++++++++++++ 3 files changed, 282 insertions(+), 39 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index dfac054b..1335d8d6 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -277,15 +277,17 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// /// The objects could not read while the chain is read because they run past - /// , with the integer the cut left of each, or null, or null while none did: - /// what they hold is the guard's to hide, and the integer only a place to look for endstream at (#224), read once - /// however many sections ask. Let go with . + /// , each with the integer the cut left of its value, or + /// when it left none; null while none did. What they hold is the guard's to hide, and the integer + /// only a place to look for endstream at (#224), read once however many sections ask. One whose value the cut left + /// a whole reference is read through, and is not among them. Let go with . /// private Dictionary? _cutWhileChainIsRead; /// - /// The objects a guard cut that a cross-reference stream's values led to, which the chain could not read and ended at: - /// each is loaded once the chain is read, so that its guard is reached where a load reaches it (#263). + /// The objects a guard cut that a cross-reference stream's values led to, which the chain could not read and ended at, and + /// those it read through, the cut having left their value a whole reference: each is loaded once the chain is read, so + /// that its guard is reached where a load reaches it (#263). /// private List? _chainValuesCut; @@ -717,8 +719,9 @@ public PdfObject GetObject(PdfObjectId id) /// /// The object is read within , as a load reads it; one the guard cuts is /// left unread, and the guard is neither reported nor counted: the chain's sections are cut only by their own windows - /// and decoding, and a guard that holds is reached when the object is loaded once the chain is read. The objects read - /// through it nest as loads do, within . + /// and decoding, and a guard that holds is reached when the object is loaded once the chain is read. One whose value + /// the cut left a whole reference reads as that reference, as a load reads through it, and is loaded once the chain is + /// read for its guard to be reached. The objects read through it nest as loads do, within . /// /// private PdfObject? ReadWithoutLoading(PdfObjectId id) @@ -746,13 +749,20 @@ public PdfObject GetObject(PdfObjectId id) return null; } - var read = TryReadUnrecorded(id, PositionOf(entry), out var value, out var pastGuard) ? value : null; + // A reference the cut left whole leads on, as a load reads through it (#224): the guard hides nothing the chain + // needs of the object, and is reached once the chain is read, where loading the object reaches it. + var parsed = TryReadUnrecorded(id, PositionOf(entry), out var value, out var pastGuard); + var read = parsed || (pastGuard && value is PdfReference) ? value : null; (_readWhileChainIsRead ??= [])[id] = read; - if (pastGuard) + if (pastGuard && read is null) { (_cutWhileChainIsRead ??= [])[id] = value.AsInteger() is null ? PdfNull.Instance : value; } + else if (pastGuard) + { + (_chainValuesCut ??= []).Add(id); + } return read; } @@ -2264,8 +2274,8 @@ private void CheckDeferredLengths() /// /// Loads, once the chain is read, each object a guard cut that a cross-reference stream's value led to, which the chain - /// ended at: the guard is reached, and reported, where a load reaches it — and thrown, when the caller asked for it —, - /// as for a /Length read once the chain is (#263). + /// ended at or read through: the guard is reached, and reported, where a load reaches it — and thrown, when the caller + /// asked for it —, as for a /Length read once the chain is (#263). /// private void ReachChainValuesCut() { @@ -3890,10 +3900,10 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect /// where that row places it as reads one: nothing is corrected, cached, recorded or /// reported. One a guard cuts gives what the cut left of it, an integer read before the cut only a place to look for /// endstream at (#224), and says so, whether an endstream confirms it or not: - /// rows the data taken again still lacks are the guard's. So does the object a guard cut at the end of a chain of - /// references from the object read, or from the /Length when a newer section gave the number: the integer the cut - /// left of that object, read where the sections read so far place it, is the place to look at then - /// (). A row that does not stand otherwise — a newer section gave + /// rows the data taken again still lacks are the guard's. A reference it holds, which a cut after it leaves whole, leads + /// on, as does the /Length when a newer section gave the number: to a length an object read whole holds, the + /// file's, or to the integer the cut left of an object a guard cut, read where the sections read so far place it, only a + /// place to look at then (). A row that does not stand otherwise — a newer section gave /// the number, or refused it; the row lies past those held, places the object in an object stream, or under another /// generation — leaves the data as it was taken. /// @@ -3915,7 +3925,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // The object a newer section places, which the chain could not read: a guard that cut it, or the object a chain // of references from it leads to, leaves how far the data runs to the integer the cut left, where an endstream // confirms it, and unknown otherwise. - return TakeAgainAtCut(stream, deferred, stream.Dictionary.GetRaw(PdfName.Length), LengthLinks, out lengthCut); + return TakeAgainThrough(stream, deferred, stream.Dictionary.GetRaw(PdfName.Length), LengthLinks, out lengthCut); } // The rows that number the object, in the order the subsections give them, up to the first of a type the reading @@ -3954,24 +3964,27 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect return null; } - var read = TryReadUnrecorded(deferred.Length, PositionOf((long)offset), out var value, out var pastGuard); - - // Data taken again to an integer a guard's cut left is no length the file gave: rows it still lacks are the guard's. - lengthCut = pastGuard; + if (!TryReadUnrecorded(deferred.Length, PositionOf((long)offset), out var value, out var pastGuard) && !pastGuard) + { + return null; + } - if ((read || pastGuard) && TakeAgain(stream, deferred, value) is { } whole) + // The object read holds the chain's second link, which a cut after it leaves whole: the chain goes on as far as the + // parse resolves a /Length through. + if (value is PdfReference) { - return whole; + return TakeAgainThrough(stream, deferred, value, LengthLinks - 1, out lengthCut); } - // The object read holds the chain's second link: the chain goes on as far as the parse resolves a /Length through. - return !pastGuard && value is PdfReference ? TakeAgainAtCut(stream, deferred, value, LengthLinks - 1, out lengthCut) : null; + // Data taken again to an integer a guard's cut left is no length the file gave: rows it still lacks are the guard's. + lengthCut = pastGuard; + return TakeAgain(stream, deferred, value); } /// /// Gives cross-reference stream with its data taken again to the length - /// gives, when it gives more than the data taken and an endstream confirms it; null - /// otherwise. + /// , an integer, gives, when it gives more than the data taken and an endstream confirms + /// it; null otherwise. /// private PdfStream? TakeAgain(PdfStream stream, DeferredLength deferred, PdfObject value) { @@ -3984,25 +3997,45 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect } /// - /// Gives cross-reference stream with its data taken again, when , read - /// through the sections read so far and references, leads to an object a guard cut: to the - /// integer the cut left of it, read where they place it, when an endstream confirms it (#224). Null otherwise, - /// and says whether a guard cut that object, which leaves how far the data runs unknown, the - /// data taken again or not. + /// Gives cross-reference stream with its data taken again to the length + /// leads to, read through the sections read so far and references, + /// as takes it; null when the chain reaches no length. /// - private PdfStream? TakeAgainAtCut(PdfStream stream, DeferredLength deferred, PdfObject? written, int links, out bool lengthCut) + /// + /// An object a guard cut ends the chain at the integer the cut left of it, read once, where a section read so far places + /// it (#224): only a place to look for endstream at, which says, the data taken again + /// or not — rows it still lacks are the guard's. One whose value the cut left a whole reference is read through, as a load + /// reads through it (). A chain that reaches no value — an object no section read so far + /// places, or a reference past the links — leaves the data as it was taken, as a whole read leaves it. + /// + private PdfStream? TakeAgainThrough(PdfStream stream, DeferredLength deferred, PdfObject? written, int links, out bool lengthCut) { - Follow(PdfName.Length, written, entry: true, out _, out var cut, links); - lengthCut = cut is not null; + lengthCut = false; + var value = written; - // What the cut left of the object, which ReadWithoutLoading read where a section read so far places it, is kept with - // it: it is not read again for each section that asks. - if (cut is not { } id) + for (var link = 0; value is PdfReference reference; link++) { - return null; + if (link == links) + { + return null; + } + + if (ReadWithoutLoading(reference.Id) is { } read) + { + value = read; + } + else if (_cutWhileChainIsRead?.TryGetValue(reference.Id, out var left) == true) + { + value = left; + lengthCut = true; + } + else + { + return null; + } } - return TakeAgain(stream, deferred, _cutWhileChainIsRead![id]); + return value is null ? null : TakeAgain(stream, deferred, value); } /// diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 86de386e..ec887346 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -528,6 +528,105 @@ public static byte[] UpdatePlacingAPartedLength() return file.ToArray(); } + /// + /// A file of one cross-reference stream, object 7, whose /Length names object 5, which only the stream's own row + /// places, before the rows of objects 1 and 2; two rows of reserved types spell endstream after an end-of-line + /// after them, 28 and 42 bytes in, before the rows of objects 3 and 7 and fifty free ones, 420 bytes in all. Object 5 + /// holds the length after 1,014 spaces — a MaxObjectLength of 1,024 parts it after its 42 —, or, with + /// , 6 0 R and 1,100 spaces after it — the same bound cuts it after the whole + /// reference —, object 6 holding the length, which a newer classic table indexes and names the stream through + /// /Prev from. The file is valid, unless : the stream's /Index declares one + /// row more than its data holds. + /// + public static byte[] OwnRowPlacingAPaddedLength(bool throughReference, bool lackingRow = false) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + const int Free = 50; + var spelled = "\nendstream\0\0\0\0"u8.ToArray(); + + // The data's length does not depend on the offsets its rows hold. + byte[] Data(long five, long stream) + { + var data = new List(Encode([(0, 0, 65535), (1, five, 0), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0)], Rows.Raw)); + data.AddRange(spelled); + data.AddRange(spelled); + data.AddRange(Encode([(1, file.OffsetOf(3), 0), (1, stream, 0), .. Enumerable.Repeat((0, 0L, 0), Free)], Rows.Raw)); + return [.. data]; + } + + var length = Data(0, 0).Length.ToString(CultureInfo.InvariantCulture); + + if (throughReference) + { + file.Object(6, length); + } + + file.Object(5, throughReference ? "6 0 R" + new string(' ', 1100) : new string(' ', 1014) + length); + var stream = file.Position; + var size = 30 + Free + (lackingRow ? 1 : 0); + file.Stream( + 7, + string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size {size} /W [1 4 2] /Index [0 1 5 1 1 2 20 2 22 2 3 1 7 1 30 {size - 30}] /Root 1 0 R /Length 5 0 R"), + Data(file.OffsetOf(5), stream)); + + if (!throughReference) + { + file.StartXRef(stream); + return file.ToArray(); + } + + file.Text("%" + new string('-', 1200) + "\n"); + var xref = file.Position; + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"xref\n0 1\n0000000000 65535 f\r\n6 1\n{file.OffsetOf(6):D10} 00000 n\r\ntrailer\n<< /Size {size} /Root 1 0 R /Prev {stream} >>\nstartxref\n{xref}\n%%EOF\n")); + return file.ToArray(); + } + + /// + /// A file of three revisions. The first holds the catalog, the page tree and the page, object 5, which holds + /// 6 0 R and 1,100 spaces after it — a MaxObjectLength of 1,024 cuts it after the whole reference —, + /// object 6, which holds 49, and cross-reference stream 7, whose /Length names object 5 and whose rows of objects + /// 0 to 2 and 7 come before two rows of reserved types that spell endstream after an end-of-line, before the + /// page's row: 49 bytes. The second is cross-reference stream 20, of 49 bytes too, which indexes objects 6 and 20 to 25, + /// and names stream 7 through /Prev; its /Length is 49, or, with , + /// 5 0 R, which the chain then reads before any section places object 6. The third is a classic table that + /// indexes object 5 and names stream 20 through /Prev. The file is valid, unless : + /// stream 7's /Index declares one row more than its data holds. + /// + public static byte[] UpdatesReachingALengthThroughACutReference(bool bothNameIt, bool lackingRow = false) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + file.Object(5, "6 0 R" + new string(' ', 1100)); + file.Object(6, "49"); + var older = file.Position; + var data = new List(Encode([(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, older, 0)], Rows.Raw)); + data.AddRange("\nendstream\0\0\0\0"u8.ToArray()); + data.AddRange(Encode([(1, file.OffsetOf(3), 0)], Rows.Raw)); + file.Stream( + 7, + string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size {(lackingRow ? 11 : 10)} /W [1 4 2] /Index [0 3 7 1 8 2 3 {(lackingRow ? 2 : 1)}] /Root 1 0 R /Length 5 0 R"), + [.. data]); + var newer = file.Position; + var rows = Encode([(1, file.OffsetOf(6), 0), (1, newer, 0), .. Enumerable.Repeat((0, 0L, 0), 5)], Rows.Raw); + file.Stream( + 20, + string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size 26 /W [1 4 2] /Index [6 1 20 6] /Root 1 0 R /Prev {older} /Length {(bothNameIt ? "5 0 R" : "49")}"), + rows); + file.Text("%" + new string('-', 1200) + "\n"); + var xref = file.Position; + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"xref\n0 1\n0000000000 65535 f\r\n5 1\n{file.OffsetOf(5):D10} 00000 n\r\ntrailer\n<< /Size 26 /Root 1 0 R /Prev {newer} >>\nstartxref\n{xref}\n%%EOF\n")); + return file.ToArray(); + } + /// /// A file as writes it, the rows that spell endstream numbering objects 8 /// and 9, but for a length a chain of objects gives: stream 7's /Length names object 5 diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index a5e2cef0..d33f9270 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -628,6 +628,117 @@ public void Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_t new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().BeFalse(); } + [Fact] + public void Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s() + { + // As above, but the stream's own row places object 5, whose 420 MaxObjectLength parts after its 42, which the rows' + // second spelled endstream confirms: the 54 rows the data taken again lacks are the guard's, not the file's (#224). + var file = ChainFiles.OwnRowPlacingAPaddedLength(throughReference: false); + + using (var whole = PdfDocument.Open(file)) + { + whole.Diagnostics.Should().BeEmpty(); + whole.GetObject(new PdfObjectId(3)).AsDictionary().Should().NotBeNull(); + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.Reader.Structure.Sections[0].CutByLimit.Should().BeTrue(); + document.Reader.Structure.Sections[0].Incomplete.Should().BeFalse(); + new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().BeFalse(); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it(bool lackingRow) + { + // The stream's own row places object 5, which MaxObjectLength cuts after "6 0 R", whole; object 6, which a newer + // section places, holds the length. The data is taken again to it, as read whole: the guard hid nothing the length + // needs, and the rows the data lacks — one, where the /Index declares one more — are the file's, as read whole. They + // went to the guard (#224). + var file = ChainFiles.OwnRowPlacingAPaddedLength(throughReference: true, lackingRow); + using var whole = PdfDocument.Open(file); + using var document = PdfDocument.Open(file, ObjectGuard); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + whole.Diagnostics.Should().BeEmpty(); + + foreach (var read in (ReadOnlySpan)[whole, document]) + { + var stream = read.Reader.Structure.Sections[1]; + stream.CutByLimit.Should().BeFalse(); + stream.Incomplete.Should().Be(lackingRow); + read.GetObject(new PdfObjectId(3)).AsDictionary().Should().NotBeNull(); + new PdfValidator().Validate(read).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().Be(lackingRow); + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public void A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first(bool bothNameIt, bool lackingRow) + { + // Object 5 holds "6 0 R", which MaxObjectLength cuts after; object 6, which the newer stream places, holds the length + // the older stream's /Length gives through 5. With the newer stream's /Length naming 5 too, the chain reads 5 before + // any section places 6, and what the cut left of it was kept as that read judged it: the older stream's data could + // not be taken again, its rows past the spelled endstream went to the guard and the file was said to give a length + // no endstream confirms; with a row lacking, the section was cut where it is the file's to lack (#224). The guard hides + // nothing either length needs: each reads as read whole. + var file = ChainFiles.UpdatesReachingALengthThroughACutReference(bothNameIt, lackingRow); + using var whole = PdfDocument.Open(file); + using var document = PdfDocument.Open(file, ObjectGuard); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + whole.Diagnostics.Should().BeEmpty(); + + foreach (var read in (ReadOnlySpan)[whole, document]) + { + var sections = read.Reader.Structure.Sections; + sections.Should().HaveCount(3); + sections.Should().AllSatisfy(section => section.CutByLimit.Should().BeFalse()); + sections[2].Incomplete.Should().Be(lackingRow); + read.GetObject(new PdfObjectId(3)).AsDictionary().Should().NotBeNull(); + new PdfValidator().Validate(read).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().Be(lackingRow); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard(bool throwOnLimit) + { + // Stream 7's /W names object 5, which MaxObjectLength cuts after "9 0 R", whole; object 9, which the newer table + // places, holds [1 4 2]. The chain ended at object 5 for the guard, the section cut before its first row, though the + // guard hid nothing the value needs. It reads through, as a load reads through it, and reaches the guard once the + // chain is read: reported, or thrown at opening when the caller asks for it (#263). + var file = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W 5 0 R /Length {length}", + "9 0 R" + new string(' ', 1100), + compressed: false, + predicted: false, + placed: true, + placedToo: new Dictionary { [9] = "[1 4 2]" }); + + if (throwOnLimit) + { + var opening = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); + opening.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); + return; + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.Reader.Structure.Sections.Should().HaveCount(2).And.OnlyContain(section => !section.CutByLimit); + document.GetObject(new PdfObjectId(3)).AsDictionary().Should().NotBeNull(); + } + [Fact] public void Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value() { From 76d923966753dcf100573b78bfffb99c27cd031b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 19:22:07 +0000 Subject: [PATCH 070/146] fix: leave to the guard only how a deferred length's data ends, not the faults it holds A cross-reference stream whose /Length the chain could not read has its data taken up to the first endstream it holds. Where a guard keeps that data from being taken again, or a guard's cut of the /Length gives where it ends, that end is the guard's to hide. Every report decoding made beside the guard was dropped, though a fault in the bytes a filter read, which the whole data meets at the same place, is the file's: a stray byte in ASCIIHex data went unreported, where the same stream with a direct /Length reports it. A limit report the capacity had dropped went too, neither kept nor counted. And data taken again to an integer a guard's cut left, or left at the first endstream with that /Length cut, reported a Flate run out at its end as the file's fault. The data is now decoded once more, knowing that its end is hidden, as data a guard cut is. A filter that runs out there says nothing of the file, and what it met before stands. That second decoding does not count the data as cut. It runs only where a guard hides the end, and no more than once. Refs #220, #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../Diagnostics/PdfDiagnostics.cs | 37 --------- .../IO/Filters/PdfFilterPipeline.cs | 44 ++++++++--- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 27 ++++--- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 76 ++++++++++++++++--- .../DocumentReaderTests.cs | 43 +++++++++++ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 23 ++++++ 6 files changed, 180 insertions(+), 70 deletions(-) diff --git a/src/AdCodicem.Pdf/Diagnostics/PdfDiagnostics.cs b/src/AdCodicem.Pdf/Diagnostics/PdfDiagnostics.cs index 68888d1a..974e0899 100644 --- a/src/AdCodicem.Pdf/Diagnostics/PdfDiagnostics.cs +++ b/src/AdCodicem.Pdf/Diagnostics/PdfDiagnostics.cs @@ -129,43 +129,6 @@ internal void RollBack(PdfDiagnosticsMark mark) } } - /// - /// Drops every entry recorded since but the guards reached, their limit.* reports and - /// their count standing: what a reading a guard stopped short met beside the guard is no fault the file holds. The - /// reports decodings moved in that the mark recorded are held again, with the guards' kept, so that a decoding that - /// meets one again does not report it twice. - /// - internal void RollBackBesideLimits(PdfDiagnosticsMark mark) - { - var kept = mark.Count; - var held = mark.DecodingReports is { } recorded ? new Dictionary(recorded) : null; - - for (var i = mark.Count; i < _entries.Count; i++) - { - var entry = _entries[i]; - - if (!entry.Code.StartsWith("limit.", StringComparison.Ordinal)) - { - continue; - } - - _entries[kept++] = entry; - - if (held is not null && _decodingReports?.ContainsKey(entry) == true) - { - held[entry] = held.GetValueOrDefault(entry) + 1; - } - } - - _entries.RemoveRange(kept, _entries.Count - kept); - _suppressed = mark.Suppressed; - - if (held is not null) - { - _decodingReports = held; - } - } - /// /// Moves every entry recorded since into , which applies /// its own capacity, and drops them from this instance. diff --git a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs index a13d95a9..ed8844d4 100644 --- a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs +++ b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs @@ -33,7 +33,7 @@ public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diag /// /// Bounds what each filter's output may be and how many filters run, and says what reaching either does. public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diagnostics, PdfLimitGuard guard) => - Decode(stream, diagnostics, guard, out _, out _); + Decode(stream, diagnostics, guard, endHidden: false, out _, out _); /// /// Decodes a stream's data under its document's guards, and says whether one of them ended decoding — the most filters @@ -56,10 +56,30 @@ public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diag /// Decoding reaches one of its document's guards, and the document was opened to throw when it does. /// public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diagnostics, out bool leftEncoded, out bool cutByGuard) => - Decode(stream, diagnostics, stream.Data.LimitGuard, out leftEncoded, out cutByGuard); + Decode(stream, diagnostics, stream.Data.LimitGuard, endHidden: false, out leftEncoded, out cutByGuard); + + /// + /// Decodes a stream's data as does, its end, with + /// , one a guard hides: a filter that runs out there says nothing of the file, as for data a + /// guard cut, while a fault in what it read is the file's still. + /// + /// The stream to decode. + /// Receives what decoding met; when none are supplied, the stream's document does. + /// + /// Whether where the data ends is the guard's to hide: data a search took up to an endstream nothing confirmed, + /// the length that would say a guard's. The data is not counted as cut by it. + /// + /// As the other overload gives it. + /// As the other overload gives it. + /// + /// Decoding reaches one of its document's guards, and the document was opened to throw when it does. + /// + public static ReadOnlyMemory Decode( + PdfStream stream, PdfDiagnostics? diagnostics, bool endHidden, out bool leftEncoded, out bool cutByGuard) => + Decode(stream, diagnostics, stream.Data.LimitGuard, endHidden, out leftEncoded, out cutByGuard); private static ReadOnlyMemory Decode( - PdfStream stream, PdfDiagnostics? diagnostics, PdfLimitGuard guard, out bool leftEncoded, out bool cutByGuard) + PdfStream stream, PdfDiagnostics? diagnostics, PdfLimitGuard guard, bool endHidden, out bool leftEncoded, out bool cutByGuard) { diagnostics ??= guard.DocumentDiagnostics; @@ -83,7 +103,7 @@ private static ReadOnlyMemory Decode( guard.Reach(PdfLimit.StreamLength, diagnostics, guard.DescribeStreamLengthCut(), stream.Data.Position); } - return DecodeChain(stream, chain, data, diagnostics, guard, out leftEncoded, out cutByGuard); + return DecodeChain(stream, chain, data, diagnostics, guard, endHidden, out leftEncoded, out cutByGuard); } var decodings = diagnostics.Decodings; @@ -91,7 +111,7 @@ private static ReadOnlyMemory Decode( try { - return DecodeChain(stream, chain, data, decodings, guard, out leftEncoded, out cutByGuard); + return DecodeChain(stream, chain, data, decodings, guard, endHidden, out leftEncoded, out cutByGuard); } finally { @@ -107,15 +127,17 @@ private static ReadOnlyMemory DecodeChain( ReadOnlyMemory data, PdfDiagnostics? diagnostics, PdfLimitGuard guard, + bool endHidden, out bool leftEncoded, out bool cutByGuard) { leftEncoded = false; - // Whether the data a filter is given is data the reader cut at one of its limits: MaxStreamLength, or the window a - // guard stopped at, cut the stream's data, or a filter before it stopped at MaxDecodedStreamLength. Every filter - // after a cut is given cut data, and the cut is reported once, by the guard that made it. - var cut = stream.Data.CutByGuard; + // Whether the data a filter is given ends where the file's may not: data the reader cut at one of its limits — + // MaxStreamLength, or the window a guard stopped at, cut the stream's data, or a filter before it stopped at + // MaxDecodedStreamLength —, or data whose end a guard hides. Every filter after it is given such data, and a cut is + // reported once, by the guard that made it; data whose end is hidden is no cut. + var cut = stream.Data.CutByGuard || endHidden; var limitReported = false; // The filters before the last share one bound for what they pass along together, and the last keeps its own: a @@ -151,7 +173,7 @@ private static ReadOnlyMemory DecodeChain( // library does not know: the filters after it would read what it encoded, and each fail on it again. if (failed) { - cutByGuard = cut; + cutByGuard = stream.Data.CutByGuard || limitReported; return data; } @@ -163,7 +185,7 @@ private static ReadOnlyMemory DecodeChain( // A value a guard cut or left out of the index was reported where the guard was reached: the chain ends there in // silence. leftEncoded = chain.End is FilterChainEnd.Limit or FilterChainEnd.Guard; - cutByGuard = cut; + cutByGuard = stream.Data.CutByGuard || limitReported; return data; } diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 1335d8d6..2abf5b47 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3760,9 +3760,12 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // Whether a guard cut the /Length the section's own row places, which leaves how far the data runs unknown. var lengthCut = false; + // Whether where the data ends is the guard's to hide, which the data is decoded knowing. + var endHidden = false; + while (true) { - var decoded = Filters.PdfFilterPipeline.Decode(xrefStream, _diagnostics, out var leftEncoded, out var cut); + var decoded = Filters.PdfFilterPipeline.Decode(xrefStream, _diagnostics, endHidden, out var leftEncoded, out var cut); // Data the guard on filters left encoded holds none of the section's rows: the section is cut before its first. data = leftEncoded ? default : decoded.Span; @@ -3786,7 +3789,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // where the section's own row for that /Length, among those held, gives more data, which an endstream confirms, // the rows spelled an endstream, and the data is taken again to that length (#220). Data that holds no row whole // is asked too: a newer section may place the object, which a guard cut. - if (!retaken && deferred is { } unsettled && declared > held && decodedWhole) + if (!retaken && !endHidden && deferred is { } unsettled && declared > held && decodedWhole) { if (RetakeDeferred(xrefStream, unsettled, data, fieldWidths, row, held, index is null ? [0, declared] : subsections, out lengthCut) is { } whole) { @@ -3794,19 +3797,23 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect xrefStream = whole; deferred = unsettled with { Taken = whole.Data.Length }; retaken = true; + endHidden = lengthCut; continue; } } - break; - } + // Data a /Length the chain could not read left to an endstream nothing confirmed, which a guard on decoding kept + // from being taken again to its length, or taken to what a guard's cut left of the length: where it ends is the + // guard's, which a filter run over data the endstream its rows spell cut short says nothing of the file at. The + // data is decoded once more knowing it, as data a guard cut is: a fault in what a filter read stands (#220, #224). + if (!endHidden && deferred is not null && !xrefStream.Data.CutByGuard && (lengthCut || (!retaken && !decodedWhole))) + { + _diagnostics.RollBack(decoding); + endHidden = true; + continue; + } - // Data a /Length the chain could not read left to an endstream nothing confirmed, which a guard on decoding kept from - // being taken again to its length: what decoding it met beside the guard — a filter run over data the endstream its - // rows spell cut short — is no fault the file holds, and only the guard's report stands (#220). - if (deferred is not null && !retaken && !decodedWhole) - { - _diagnostics.RollBackBesideLimits(decoding); + break; } section.CutByLimit |= !decodedWhole; diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index ec887346..dcb34b7f 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -502,29 +502,81 @@ public static byte[] UpdatePlacingAPaddedLength( /// A file of two revisions. The first holds the catalog, the page tree and the page, object 5, which holds the data's /// length, 210, after 1,014 spaces — a MaxObjectLength of 1,024 parts it after its 21 —, and cross-reference /// stream 7, whose /Length names object 5 and whose rows spell endstream after an end-of-line twice, 7 and - /// 21 bytes in, before the rows of objects 1 to 3, 5 and 7 and twenty free ones. The second is a classic table that - /// indexes object 5 again, where it is, and names stream 7 through /Prev. The file is valid. + /// 21 bytes in, before the rows of objects 1 to 3, 5 and 7 and free ones. With + /// , the rows are compressed in a stored block, which shows them as written 7 bytes in: the + /// data is 284 bytes with 29 free rows, which the same bound parts after its 28, where the second endstream is, + /// and 221 with 20, parted after its 22, where none is. The second is a classic table that indexes object 5 again, where + /// it is, and names stream 7 through /Prev. The file is valid. /// - public static byte[] UpdatePlacingAPartedLength() + public static byte[] UpdatePlacingAPartedLength(int free = 20, bool compressed = false) { using var file = new Writer("1.5"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); - file.Object(5, new string(' ', 1014) + "210"); + + // The data's length does not depend on the offsets its rows hold. + byte[] Data(long five, long stream) + { + var spelled = "\nendstream\0\0\0\0"u8.ToArray(); + var data = new List(Encode([(0, 0, 65535)], Rows.Raw)); + data.AddRange(spelled); + data.AddRange(spelled); + data.AddRange(Encode([(1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), (1, five, 0), (1, stream, 0)], Rows.Raw)); + data.AddRange(Encode([.. Enumerable.Repeat((0, 0L, 0), free)], Rows.Raw)); + return compressed ? Compress([.. data]) : [.. data]; + } + + file.Object(5, new string(' ', 1014) + Data(0, 0).Length.ToString(CultureInfo.InvariantCulture)); var stream = file.Position; - var spelled = "\nendstream\0\0\0\0"u8.ToArray(); - var data = new List(Encode([(0, 0, 65535)], Rows.Raw)); - data.AddRange(spelled); - data.AddRange(spelled); - data.AddRange(Encode([(1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), (1, file.OffsetOf(5), 0), (1, stream, 0)], Rows.Raw)); - data.AddRange(Encode([.. Enumerable.Repeat((0, 0L, 0), 20)], Rows.Raw)); - file.Stream(7, "/Type /XRef /Size 50 /W [1 4 2] /Index [0 1 20 2 22 2 1 3 5 1 7 1 30 20] /Root 1 0 R /Length 5 0 R", [.. data]); + var size = 30 + free; + file.Stream( + 7, + string.Create( + CultureInfo.InvariantCulture, + $"/Type /XRef /Size {size} /W [1 4 2] /Index [0 1 20 2 22 2 1 3 5 1 7 1 30 {free}]{(compressed ? " /Filter /FlateDecode" : string.Empty)} /Root 1 0 R /Length 5 0 R"), + Data(file.OffsetOf(5), stream)); + file.Text("%" + new string('-', 1200) + "\n"); + var xref = file.Position; + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"xref\n0 1\n0000000000 65535 f\r\n5 1\n{file.OffsetOf(5):D10} 00000 n\r\ntrailer\n<< /Size {size} /Root 1 0 R /Prev {stream} >>\nstartxref\n{xref}\n%%EOF\n")); + return file.ToArray(); + } + + /// + /// A file of two revisions. The first holds the catalog, the page tree and the page, object 4, and cross-reference stream + /// 5, whose rows of objects 0 to 5, none spelling endstream, are compressed in a stored block and written in + /// hexadecimal after a stray x, which encodes nothing: /Filter [/ASCIIHexDecode /FlateDecode]. Its + /// /Length names object 4, which holds it and which only the stream indexes — or, unless + /// , is written direct. The second is a classic table that indexes objects 1 to 3 and names + /// stream 5 through /Prev. + /// + public static byte[] UpdateOverAStrayHexByte(bool deferred) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + + // The data's length does not depend on the offsets its rows hold. + byte[] Data(long four, long stream) => Encoding.ASCII.GetBytes( + "x" + Convert.ToHexString(Compress(Encode( + [(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), (1, four, 0), (1, stream, 0)], + Rows.Raw))) + ">"); + + var length = Data(0, 0).Length.ToString(CultureInfo.InvariantCulture); + file.Object(4, length); + var at = file.Position; + file.Stream( + 5, + $"/Type /XRef /Size 6 /W [1 4 2] /Filter [/ASCIIHexDecode /FlateDecode] /Root 1 0 R /Length {(deferred ? "4 0 R" : length)}", + Data(file.OffsetOf(4), at)); file.Text("%" + new string('-', 1200) + "\n"); var xref = file.Position; file.Text(string.Create( CultureInfo.InvariantCulture, - $"xref\n0 1\n0000000000 65535 f\r\n5 1\n{file.OffsetOf(5):D10} 00000 n\r\ntrailer\n<< /Size 50 /Root 1 0 R /Prev {stream} >>\nstartxref\n{xref}\n%%EOF\n")); + $"xref\n0 4\n0000000000 65535 f\r\n{file.OffsetOf(1):D10} 00000 n\r\n{file.OffsetOf(2):D10} 00000 n\r\n{file.OffsetOf(3):D10} 00000 n\r\ntrailer\n<< /Size 6 /Root 1 0 R /Prev {at} >>\nstartxref\n{xref}\n%%EOF\n")); return file.ToArray(); } diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 876176c0..957c16fe 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -1943,6 +1943,49 @@ public void A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_ document.Diagnostics.Count(entry => entry.Code == PdfDiagnosticCodes.LimitFilterChain).Should().Be(1); } + [Theory] + [InlineData(1)] + [InlineData(0)] + public void A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted(int capacity) + { + // As above, under a capacity with room for one report, or none: the guard's report, which the capacity dropped, went + // with the decoding's other reports, neither kept nor counted (#220). + var file = ChainFiles.AloneWithIndirectLengthBeforeSpelledRows(reservedRowFirst: false, stored: true, layers: 2); + var limits = PdfReaderLimits.Default with { MaxFilterChainLength = 1 }; + using var full = PdfDocument.Open(file, new PdfReaderOptions { Limits = limits }); + using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = limits, DiagnosticCapacity = capacity }); + + document.Diagnostics.Should().Equal(full.Diagnostics.Take(capacity)); + (document.Diagnostics.Count + document.Diagnostics.SuppressedCount).Should().Be(full.Diagnostics.Count); + } + + [Theory] + [InlineData(nameof(PdfReaderLimits.MaxFilterChainLength))] + [InlineData(nameof(PdfReaderLimits.MaxDecodedStreamLength))] + [InlineData(nameof(PdfReaderLimits.MaxStreamLength))] + public void A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one(string limit) + { + // Stream 5's hexadecimal data starts with a stray x, which encodes nothing, and its rows spell no endstream. Read + // whole, the fault is the file's, whether the /Length names object 4, which only the stream places, or is written + // direct. Under a guard on decoding, or MaxStreamLength, only the guard's report stood when the /Length waited for + // the chain: but the fault lies before the guard stops, where the whole data meets it, and only how the data ends is + // the guard's to hide (#220). + var limits = limit switch + { + nameof(PdfReaderLimits.MaxFilterChainLength) => PdfReaderLimits.Default with { MaxFilterChainLength = 1 }, + nameof(PdfReaderLimits.MaxDecodedStreamLength) => PdfReaderLimits.Default with { MaxDecodedStreamLength = 20 }, + _ => PdfReaderLimits.Default with { MaxStreamLength = 40 }, + }; + using var direct = PdfDocument.Open(ChainFiles.UpdateOverAStrayHexByte(deferred: false), new PdfReaderOptions { Limits = limits }); + using var deferred = PdfDocument.Open(ChainFiles.UpdateOverAStrayHexByte(deferred: true), new PdfReaderOptions { Limits = limits }); + + Faults(direct).Should().ContainSingle().Which.Should().StartWith("An ASCIIHex stream holds a byte that encodes nothing"); + Faults(deferred).Should().Equal(Faults(direct)); + + static string[] Faults(PdfDocument document) => + [.. document.Diagnostics.Where(entry => entry.Code == PdfDiagnosticCodes.FilterFailed).Select(entry => entry.Message)]; + } + [Fact] public void A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is() { diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index d33f9270..56bc5ac1 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -628,6 +628,29 @@ public void Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_t new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().BeFalse(); } + [Theory] + [InlineData(29)] + [InlineData(20)] + public void A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file(int free) + { + // As above, the rows compressed in a stored block. With 29 free rows the data is 284 bytes, which MaxObjectLength + // parts after its 28, where the rows' second spelled endstream confirms it: the data is taken again to it; with 20, + // 221, parted after its 22, which nothing confirms: the data stays at the first spelled endstream. Either way it ends + // where the guard hides the length, and the Flate run out there was reported as the file's fault (#224). + var file = ChainFiles.UpdatePlacingAPartedLength(free, compressed: true); + + using (var whole = PdfDocument.Open(file)) + { + whole.Diagnostics.Should().BeEmpty(); + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.Reader.Structure.Sections[1].CutByLimit.Should().BeTrue(); + } + [Fact] public void Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s() { From fafefd60a93608562c93b8618c532a3cb814142b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 19:35:41 +0000 Subject: [PATCH 071/146] fix: take an R a window's edge reaches for a reference only where the file ends it there A window MaxObjectLength ends right after the R of "6 0 Rx" or "6 0 RG" left the parser an R it took whole: the cut object held a reference the file never wrote. A value read through it, a /Filter or a /Length, followed that reference to its target, past the object the guard cut, and was read as the file's: the data decoded, or a length judged. An R the edge reaches is now asked of the file, as an endobj is. Where a regular byte goes on, the edge cut what may be a reference, as under a bound one byte smaller, and only what lies before it is read. A parser no file backs asks for a larger window, as for any token the edge may have cut. Refs #224, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 22 ++++++++----- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 31 +++++++++++++++++++ 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index fc9f5b43..349d5971 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -443,9 +443,10 @@ private PdfObject ParseIntegerOrReference(PdfToken token, int depth) var second = _lexer.Read(); var third = second.Kind == PdfTokenKind.Integer ? _lexer.Read() : second; - if (second.Kind == PdfTokenKind.Integer && third.IsKeyword("R"u8) && - token.Integer is >= 0 and <= PdfObjectId.MaxNumber && - second.Integer is >= 0 and <= PdfObjectId.MaxGeneration) + // An R a window's edge reaches is one only where the file says it ends there: "12 0 Rx" is no reference. + var referenced = second.Kind == PdfTokenKind.Integer && IsWholeKeyword(third, "R"u8); + + if (referenced && token.Integer is >= 0 and <= PdfObjectId.MaxNumber && second.Integer is >= 0 and <= PdfObjectId.MaxGeneration) { return new PdfReference(new PdfObjectId((int)token.Integer, (int)second.Integer), _source); } @@ -473,8 +474,8 @@ private PdfObject ParseIntegerOrReference(PdfToken token, int depth) } // The buffer ended before the look-ahead could tell: "12 0" may be all there is, or what a window's edge left of - // "12 0 R". - if (third.Kind == PdfTokenKind.EndOfInput) + // "12 0 R" — or "12 0 R" what it left of "12 0 RG", which a larger window shows (#174). + if (third.Kind == PdfTokenKind.EndOfInput || (second.Kind == PdfTokenKind.Integer && third.IsKeyword("R"u8) && !referenced)) { _truncated = true; _lookAheadCut = !_endsData; @@ -1486,8 +1487,15 @@ private void ReportCut( /// a window ends exactly with reads whole, as under a larger window, wherever its endobj comes — after its value, where a /// container it left open is, or as all it holds (#174); a parser no file backs asks for a larger buffer. /// - private readonly bool IsEndObj(PdfToken token) => - token.IsKeyword("endobj"u8) && + private readonly bool IsEndObj(PdfToken token) => IsWholeKeyword(token, "endobj"u8); + + /// + /// Determines whether is the keyword , and whole: one that reaches the + /// end of a buffer that is a window's edge may be the start of a longer keyword — endobjX, RG —, which the + /// file says when it backs the parser; a parser no file backs asks for a larger buffer. + /// + private readonly bool IsWholeKeyword(PdfToken token, ReadOnlySpan keyword) => + token.IsKeyword(keyword) && (!AtWindowEdge(token) || (!PastWindowEdge && _fileEdge is { } file && file.TokenEndsAt(_baseOffset + token.End))); /// diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 56bc5ac1..117ac68a 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -628,6 +628,37 @@ public void Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_t new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().BeFalse(); } + [Theory] + [InlineData("Rx", false)] + [InlineData("RG", false)] + [InlineData("R\n", true)] + public void An_R_a_window_s_edge_reaches_makes_a_reference_only_where_the_file_ends_it_there(string tail, bool reference) + { + // Object 4, the stream's /Filter, holds "6 0 " and the tail after 100 spaces, and MaxObjectLength ends its window + // right after the R; object 6 holds /ASCIIHexDecode. An R a regular byte goes on was taken whole: the cut object + // held a reference the file never wrote, which the /Filter followed to object 6, past the guard's cut, and the data + // was decoded. The edge cuts what may be a reference, as under a bound one byte smaller, and the data is left as it + // is, the guard's (#224, #263). + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + [4] = file.Object(4, new string(' ', 100) + "6 0 " + tail), + [6] = file.Object(6, "/ASCIIHexDecode"), + [7] = file.Object(7, "<< /Length 7 /Filter 4 0 R >>\nstream\n414243>\nendstream"), + }; + file.Table(8, rows, "/Size 8 /Root 1 0 R"); + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = "4 0 obj\n".Length + 105 } }; + using var document = PdfDocument.Open(file.Bytes, options); + + var data = document.GetObject(new PdfObjectId(7)).AsStream().Required().Decode().ToArray(); + + Encoding.ASCII.GetString(data).Should().Be(reference ? "ABC" : "414243>"); + document.Reader.IsCutAtLimit(4).Should().BeTrue(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + } + [Theory] [InlineData(29)] [InlineData(20)] From 3696774840922cbcb0cfd35a7bb22cb2afc85af6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 19:36:06 +0000 Subject: [PATCH 072/146] fix: leave to the guard data an endstream it spells ends where the guard hid the /Length A stream whose /Length a guard cut or left out of the index is found by its endstream. One its data spells, inside a string, ends the search as the stream's own would, its object's endobj then lost: the length that tells them apart is the guard's. The data up to it was taken as the file's, not marked cut, and the object said to lack its endobj, which the validator reported of a valid file. As at a header the data spells, the data up to such an endstream, which its object's endobj does not follow, is now the guard's, cut by it, and the object's endobj is not judged. A later reading that cannot tell where an object ends takes back what an earlier one recorded of it: the chain had read a cross-reference stream whose /Length it could not read up to an endstream its rows spell. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 7 +++- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 14 ++++++- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 42 +++++++++++++++++++ 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 2abf5b47..02f81e66 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -5029,7 +5029,10 @@ private long RecordedAtFor(int number, XRefEntry entry) /// Records whether endobj follows an object read at : it does not when another /// token follows it, or when the file ends first — not when the window the reader offered ended first, where /// what follows was never seen. A stream whose data runs past the window is seen to its end all the same: the - /// file is asked what follows its endstream (#55). + /// file is asked what follows its endstream (#55). A reading that finds it, or cannot tell where the object + /// ends — a stream whose end a guard hides —, takes back what an earlier one recorded: the chain reads a + /// cross-reference stream whose /Length it cannot read up to the first endstream it holds, which its rows + /// may spell (#224). /// private void RecordEndObj(int number, int found, EndObjState state, bool cut, long offset) { @@ -5042,7 +5045,7 @@ private void RecordEndObj(int number, int found, EndObjState state, bool cut, lo { _endObjMissing[found] = offset; } - else if (state == EndObjState.Present) + else if (state is EndObjState.Present or EndObjState.Unknown) { _endObjMissing.Remove(found); } diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 349d5971..37596751 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -820,12 +820,24 @@ private PdfStream ReadStream(PdfDictionary dictionary) { // A length read once the chain is, or one a guard left out of the index or cut, is not judged: nothing is said // of it, nor a message made for it. - if (length != found && declared.Form is not (StreamLengthForm.Deferred or StreamLengthForm.ObjectUnindexed or StreamLengthForm.ObjectCut) && + var hidden = declared.Form is StreamLengthForm.ObjectUnindexed or StreamLengthForm.ObjectCut; + + if (length != found && declared.Form != StreamLengthForm.Deferred && !hidden && KeepLengthFault(Fault(declared, dataStart, found, found, EndStreamState.Found, null))) { WarnLengthFault(PdfDiagnosticCodes.StreamLengthInvalid, DescribeRecovered(declared, found), dataStart); } + // An endstream its object's endobj does not follow, where a guard cut the /Length or left it out of the index: one + // the data spells, as a valid stream's may, ends the search as the stream's own would, its object's endobj lost — + // the length that tells them apart is the guard's. The data up to it is the guard's, cut by it, the object's + // endobj unjudged, and nothing is said of the file, as at a header the data spells (#224). + if (hidden && settled.EndObj == EndObjState.Absent) + { + _endObjUnknown = true; + return Finish(dictionary, dataStart, found, span.Length, cutByGuard: true); + } + _endObjAfterStream = _streamData is null ? null : settled.EndObj; return Finish(dictionary, dataStart, found, span.Length); } diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 117ac68a..86eef3d9 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -628,6 +628,43 @@ public void Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_t new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefSectionMalformed).Should().BeFalse(); } + [Theory] + [InlineData("cut before the value")] + [InlineData("cut inside the value")] + [InlineData("left out")] + public void An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s(string guard) + { + // The content stream's data spells endstream inside a string, which its object's endobj does not follow; its own + // endstream and endobj come after the 40 bytes. Read whole, the length passes the spelled one. Where a guard cut the + // /Length, or left it out of the index, the search stops there, and the object was said to lack its endobj: the + // length that tells the two apart is the guard's, and so is the data up to it (#224). + const string Data = "BT /F1 12 Tf 72 712 Td (endstream) Tj ET"; + var (twenty, options) = guard switch + { + "cut before the value" => (new string(' ', 2000) + "40", ObjectGuard), + "cut inside the value" => (new string(' ', 1024 - "20 0 obj\n".Length - 1) + "40" + new string(' ', 100), ObjectGuard), + _ => ("40", new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxIndexEntries = 20 } }), + }; + var file = ContentLengthIn(twenty, Data); + + using (var whole = PdfDocument.Open(file)) + { + whole.GetObject(new PdfObjectId(4)).AsStream().Required().GetRawBytes().ToArray().Should().Equal(Encoding.ASCII.GetBytes(Data)); + whole.Diagnostics.Should().BeEmpty(); + } + + using var document = PdfDocument.Open(file, options); + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + stream.GetRawBytes().ToArray().Should().Equal(Encoding.ASCII.GetBytes("BT /F1 12 Tf 72 712 Td (")); + stream.Data.CutByGuard.Should().BeTrue(); + document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); + document.Reader.TryGetStreamLengthFault(4, out _).Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal( + guard == "left out" ? PdfDiagnosticCodes.LimitIndexEntries : PdfDiagnosticCodes.LimitObject); + new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.ObjectEndObjMissing).Should().BeFalse(); + } + [Theory] [InlineData("Rx", false)] [InlineData("RG", false)] @@ -1006,6 +1043,11 @@ public void Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chai section.CutByLimit.Should().BeTrue(); section.Incomplete.Should().BeFalse(); + // Loaded, the stream ends at the endstream its rows spell, which its object's endobj does not follow: the data is the + // guard's, and the object was said to lack its endobj. + document.GetObject(new PdfObjectId(7)).AsStream().Required().Data.CutByGuard.Should().BeTrue(); + document.Reader.IsEndObjMissing(7, out _).Should().BeFalse(); + var throwing = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); } From 45f103679d7f46d868d714758d1eecdda0205e32 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 19:46:52 +0000 Subject: [PATCH 073/146] fix: read no /Prev or /XRefStm of a table an /XRefStm names in a stream's place A hybrid file's stream has its own /Prev left unread since this batch: the chain goes on through the table that names the stream. An /XRefStm can name a classic table instead, which ISO 32000-1 does not allow (Table 19), and that table's /Prev and /XRefStm were read all the same. One that leads to an object MaxObjectLength cuts ended the chain for the guard, the original table and the catalog lost, and one no section places was said to name a missing section and set off a rebuild, for links the chain then threw away. They are now left unread, as a stream's own /Prev is, and the rule on trailer values judges them as written, without following a reference they hold. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 11 ++++++- .../Validation/Rules/TrailerValueWrongRule.cs | 10 ++++-- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 28 ++++++++++++++++ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 32 +++++++++++++++++++ 4 files changed, 77 insertions(+), 4 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 02f81e66..88930d77 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3101,9 +3101,18 @@ private bool TryReadClassicTable(long absolute, XRefSectionRecord section, out l { section.TrailerPosition = absolute + token.Start; - if (ReadTrailer(window.Memory, absolute, lexer.Position, windowFull, section) is { } trailer) + var trailer = ReadTrailer(window.Memory, absolute, lexer.Position, windowFull, section); + + if (trailer is not null) { MergeSectionTrailer(trailer, section); + } + + // A table an /XRefStm names, where ISO 32000-1 names a cross-reference stream (Table 19), stands in its + // place: as a hybrid file's stream's own /Prev, what its trailer names is not read, to name a section + // missing or end the chain for a guard, the chain going on through the table that names it. + if (trailer is not null && section.NamedBy != "/XRefStm") + { previous = SectionOffset(trailer, PdfName.Prev, absolute, out var previousHidden); hybrid = SectionOffset(trailer, PdfName.XRefStm, absolute, out var streamHidden); section.ChainHiddenByGuard = streamHidden ?? previousHidden; diff --git a/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs b/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs index 8fdbf406..2752fade 100644 --- a/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs +++ b/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs @@ -32,7 +32,8 @@ namespace AdCodicem.Pdf.Validation.Rules; /// the validator had the reader settle on before its first rule — the chain's own, unless a read could still have rebuilt /// it, in which case it was rebuilt then (#237) —, so that what was read before validating changes nothing. A hybrid file's /// stream's own /Prev, which the reader does not follow, the chain going on through the table's, is judged as the -/// file wrote it, a reference it holds not followed (#263). +/// file wrote it, a reference it holds not followed (#263); so are the /Prev and /XRefStm of a table an +/// /XRefStm names in a stream's place. /// /// internal sealed class TrailerValueWrongRule : IValidationRule @@ -86,14 +87,17 @@ public void Check(ValidationContext context) } else { + // A table an /XRefStm names gives values the reader does not follow either: judged as written (#263). + var followed = section.NamedBy != "/XRefStm"; + if (IsFollowed(sections, section, PdfName.Prev)) { - judge.Value("its /Prev", trailer.GetRaw(PdfName.Prev), "ISO 32000-1 makes it direct (Table 15)", "Table 15"); + judge.Value("its /Prev", trailer.GetRaw(PdfName.Prev), "ISO 32000-1 makes it direct (Table 15)", "Table 15", followed); } if (IsFollowed(sections, section, PdfName.XRefStm)) { - judge.Value("its /XRefStm", trailer.GetRaw(PdfName.XRefStm), direct: null, "Table 19"); + judge.Value("its /XRefStm", trailer.GetRaw(PdfName.XRefStm), direct: null, "Table 19", followed); } } } diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index dcb34b7f..e41dadcd 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -768,6 +768,34 @@ public static byte[] HybridWithStreamPrev(string streamPrev, int padding = 3000) return file.ToArray(); } + /// + /// A file as writes it, but for what the update's table names through + /// /XRefStm: a classic table, where ISO 32000-1 names a cross-reference stream (Table 19), which indexes object + /// 10 and whose trailer gives as . + /// + public static byte[] HybridNamingATable(string key, string value, int padding = 3000) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + var older = file.Position; + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"xref\n0 4\n0000000000 65535 f\r\n{file.OffsetOf(1):D10} 00000 n\r\n{file.OffsetOf(2):D10} 00000 n\r\n{file.OffsetOf(3):D10} 00000 n\r\ntrailer\n<< /Size 4 /Root 1 0 R >>\nstartxref\n{older}\n%%EOF\n")); + file.Object(9, new string(' ', padding) + "123"); + file.Object(10, "(ten)"); + var named = file.Position; + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"xref\n10 1\n{file.OffsetOf(10):D10} 00000 n\r\ntrailer\n<< /Size 11 /{key} {value} >>\n")); + var table = file.Position; + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"xref\n0 1\n0000000000 65535 f\r\n9 1\n{file.OffsetOf(9):D10} 00000 n\r\ntrailer\n<< /Size 11 /Root 1 0 R /Prev {older} /XRefStm {named} >>\nstartxref\n{table}\n%%EOF\n")); + return file.ToArray(); + } + /// /// A classic table updated by a cross-reference stream. The table indexes the catalog, the page tree, the page and /// object 12, the highest number; cross-reference stream 10, the update, indexes the first three with object 4, a page diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 86eef3d9..cd7f941d 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1229,6 +1229,38 @@ public void A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_throug judged.Diagnostics.Should().BeEmpty(); } + [Theory] + [InlineData("Prev", "9 0 R", true)] + [InlineData("XRefStm", "9 0 R", true)] + [InlineData("Prev", "12 0 R", false)] + [InlineData("XRefStm", "12 0 R", false)] + public void A_table_an_xrefstm_names_in_a_stream_s_place_has_its_own_links_left_unread(string key, string value, bool guarded) + { + // The update's /XRefStm names a classic table, where ISO 32000-1 names a cross-reference stream (Table 19): its rows + // are read, and the chain goes on through the update's /Prev, as past a stream. The table's own /Prev or /XRefStm, + // which a stream's is not since #263, was read all the same: one that leads to an object MaxObjectLength cuts ended + // the chain for the guard, the original table and the catalog lost, and one no section places was said to name a + // missing section. The rule on trailer values judges it as the file wrote it. + var file = ChainFiles.HybridNamingATable(key, value); + using var document = PdfDocument.Open(file, guarded ? ObjectGuard : null); + + document.WasRepaired.Should().BeFalse(); + document.Reader.Structure.Sections.Should().HaveCount(3); + document.Diagnostics.Should().BeEmpty(); + document.GetObject(new PdfObjectId(3)).AsDictionary().Should().NotBeNull(); + var report = new PdfValidator().Validate(document); + report.Contains(PdfValidationRuleIds.XRefSectionNotFound).Should().BeFalse(); + report.Contains(PdfValidationRuleIds.XRefCheckedInPart).Should().BeFalse(); + + using var judged = PdfDocument.Open(file, guarded ? ObjectGuard : null); + var trailers = new ValidationProfile("trailers", 1, [new TrailerValueWrongRule()]); + var findings = new PdfValidator(new PdfValidatorOptions { Profile = trailers }).Validate(judged).Findings.Select(finding => finding.Message); + + // A table's /XRefStm may be a reference (Table 19); its /Prev is direct (Table 15). + findings.Should().HaveCount(key == "Prev" ? 1 : 0).And.OnlyContain(message => message.Contains($"writes its /Prev as the reference {value}", StringComparison.Ordinal)); + judged.Diagnostics.Should().BeEmpty(); + } + [Theory] [InlineData("Prev")] [InlineData("XRefStm")] From b0aaeba196dd9492c5e27ea716d0c1fcc7d1c02a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 19:47:38 +0000 Subject: [PATCH 074/146] fix: follow a cross-reference stream's filter values in its trailer rule only where decoding does The rule on trailer values judges a value the reader does not read as written, since a hybrid stream's own /Prev was. Yet it followed every reference among a cross-reference stream's /Filter and /DecodeParms. That included /Columns where no predictor is, /EarlyChange under Flate, and a /DecodeParms beside no filter that reads one. It judged a real decoding never read, and reached a guard the reading never met where MaxObjectLength cuts the object such a reference names. It now follows the values decoding reads, as decoding reads them, under the reader's bound on filters, and judges the others as written. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 6 ++ .../Validation/Rules/TrailerValueWrongRule.cs | 63 +++++++++++++++---- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 32 ++++++++++ 3 files changed, 88 insertions(+), 13 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 88930d77..1847a421 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -841,6 +841,12 @@ internal ObjectPresence GetPresence(int number) /// internal bool ChainStoppedByGuard => _chainFilledIndex || _cutByGuard; + /// + /// Gets the most filters a stream's data passes through as the reader decodes it + /// (): the chain ends at the element past them. + /// + internal int FilterChainBound => _guard.Bound(PdfLimit.FilterChain); + /// /// Gets where the index first had no room for an object number, reached — /// as the document opened, or in a rebuild run since —, or -1 while it had room for every number it met. diff --git a/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs b/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs index 2752fade..f2a4babe 100644 --- a/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs +++ b/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs @@ -1,5 +1,6 @@ using System.Globalization; using AdCodicem.Pdf.IO; +using AdCodicem.Pdf.IO.Filters; using AdCodicem.Pdf.IO.XRef; using AdCodicem.Pdf.Objects; @@ -30,10 +31,13 @@ namespace AdCodicem.Pdf.Validation.Rules; /// /// Each trailer is judged as the file wrote it; a reference it holds is followed as any read follows it, through the index /// the validator had the reader settle on before its first rule — the chain's own, unless a read could still have rebuilt -/// it, in which case it was rebuilt then (#237) —, so that what was read before validating changes nothing. A hybrid file's -/// stream's own /Prev, which the reader does not follow, the chain going on through the table's, is judged as the -/// file wrote it, a reference it holds not followed (#263); so are the /Prev and /XRefStm of a table an -/// /XRefStm names in a stream's place. +/// it, in which case it was rebuilt then (#237) —, so that what was read before validating changes nothing. A value the +/// reader does not read is judged as the file wrote it, a reference it holds not followed: following it would read what +/// the reading never did, and could reach a guard it never met (#263). So are a hybrid file's stream's own /Prev, +/// which the reader does not follow, the chain going on through the table's; the /Prev and /XRefStm of a +/// table an /XRefStm names in a stream's place; and what of a cross-reference stream's /Filter and +/// /DecodeParms decoding does not read — the parameters of a filter that reads none, past its predictor where it +/// has none, or past the element that ends the chain. /// /// internal sealed class TrailerValueWrongRule : IValidationRule @@ -83,7 +87,11 @@ public void Check(ValidationContext context) } judge.Value("its /Length", trailer.GetRaw(PdfName.Length), pdf20 ? Iso2 : null, "Table 5"); - judge.Decoding(trailer, pdf20); + + // What decoding reads, followed as decoding follows it, through the bound on filters the reader read under. + var reads = new DecodingReads(); + FilterChain.TryFollow(trailer, context.Document.Reader.FilterChainBound, ref reads); + judge.Decoding(trailer, pdf20, reads); } else { @@ -192,9 +200,10 @@ public void Integers(PdfDictionary trailer, PdfName key) /// /// Reports a cross-reference stream's /Filter or /DecodeParms written as a reference, and, in a file - /// that declares PDF 2.0, each element and entry written as one; and the integers of Table 8 written as reals. + /// that declares PDF 2.0, each element and entry written as one; and the integers of Table 8 written as reals. A + /// reference among them is followed where decoding follows it, as says. /// - public void Decoding(PdfDictionary trailer, bool pdf20) + public void Decoding(PdfDictionary trailer, bool pdf20, DecodingReads reads) { var inside = pdf20 ? Iso2 : null; @@ -204,7 +213,7 @@ public void Decoding(PdfDictionary trailer, bool pdf20) var value = trailer.GetRaw(key); Value("its " + name, value, Iso1, integer: null); - switch (value?.Resolve()) + switch (reads.Resolve(value)) { case PdfArray array: for (var i = 0; i < array.Count; i++) @@ -212,27 +221,55 @@ public void Decoding(PdfDictionary trailer, bool pdf20) var what = string.Create(CultureInfo.InvariantCulture, $"element {i} of its {name}"); Value(what, array[i], inside, integer: null); - if (array[i].Resolve() is PdfDictionary parameters) + if (reads.Resolve(array[i]) is PdfDictionary parameters) { - Parameters(parameters, what, inside); + Parameters(parameters, what, inside, reads); } } break; case PdfDictionary parameters: - Parameters(parameters, "its " + name, inside); + Parameters(parameters, "its " + name, inside, reads); break; } } } - private void Parameters(PdfDictionary parameters, string where, string? inside) + private void Parameters(PdfDictionary parameters, string where, string? inside, DecodingReads reads) { foreach (var (key, value) in parameters) { - Value(FileQuote.Name(key) + " of " + where, value, inside, Array.IndexOf(IntegerParameters, key) >= 0 ? "Table 8" : null); + var integer = Array.IndexOf(IntegerParameters, key) >= 0 ? "Table 8" : null; + Value(FileQuote.Name(key) + " of " + where, value, inside, integer, reads.Follows(value)); + } + } + } + + /// + /// The references among a cross-reference stream's /Filter and /DecodeParms that decoding follows, each as + /// the file wrote it, which gives in the order decoding reads them (#263). + /// + private readonly struct DecodingReads() : IFilterValueReader + { + private readonly HashSet _followed = new(ReferenceEqualityComparer.Instance); + + /// Determines whether is no reference, or one decoding follows. + public bool Follows(PdfObject? written) => written is not PdfReference || _followed.Contains(written); + + /// Gives what leads to where decoding follows it, and itself otherwise. + public PdfObject? Resolve(PdfObject? written) => Follows(written) ? written?.Resolve() : written; + + /// + public bool TryRead(PdfName key, PdfObject? written, bool entry, out PdfObject? value) + { + if (written is PdfReference) + { + _followed.Add(written); } + + value = written?.Resolve(); + return true; } } } diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index cd7f941d..819e1b51 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1229,6 +1229,38 @@ public void A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_throug judged.Diagnostics.Should().BeEmpty(); } + [Theory] + [InlineData("/Filter /FlateDecode /DecodeParms << /Columns 5 0 R >>", true, false)] + [InlineData("/Filter /FlateDecode /DecodeParms << /EarlyChange 5 0 R >>", true, false)] + [InlineData("/DecodeParms 5 0 R", false, false)] + [InlineData("/Filter /FlateDecode /DecodeParms << /Predictor 12 /Columns 5 0 R >>", true, true)] + public void The_rule_on_trailer_values_follows_a_parameter_only_where_decoding_reads_it(string entries, bool compressed, bool read) + { + // Stream 7's /DecodeParms, or a parameter in it, is a reference to object 5, which holds 7.0. Decoding reads /Columns + // only past a predictor, /EarlyChange only for LZW, and /DecodeParms only for a Flate or LZW step: the rule followed + // each all the same, judged a real decoding never read, and reached the guard the reading never met where + // MaxObjectLength cuts object 5 (#263). Where decoding reads it, it is followed as before. + var trailers = new ValidationProfile("trailers", 1, [new TrailerValueWrongRule()]); + var file = ChainFiles.UnderAnUpdate($"/Type /XRef /Size 8 /W [1 4 2] {entries} /Length {{length}}", "7.0", compressed, predicted: read, placed: true); + using var document = PdfDocument.Open(file); + + new PdfValidator(new PdfValidatorOptions { Profile = trailers }).Validate(document).Findings + .Any(finding => finding.Message.Contains("as a real number", StringComparison.Ordinal)).Should().Be(read); + + if (read) + { + return; + } + + var padded = ChainFiles.UnderAnUpdate( + $"/Type /XRef /Size 8 /W [1 4 2] {entries} /Length {{length}}", new string(' ', 3000) + "7.0", compressed, predicted: false, placed: true); + using var guarded = PdfDocument.Open(padded, ObjectGuard); + + guarded.Diagnostics.Should().BeEmpty(); + new PdfValidator(new PdfValidatorOptions { Profile = trailers }).Validate(guarded); + guarded.Diagnostics.Should().BeEmpty(); + } + [Theory] [InlineData("Prev", "9 0 R", true)] [InlineData("XRefStm", "9 0 R", true)] From b3534fc321cc5f78970a410b5338c81da462cbc6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 19:48:34 +0000 Subject: [PATCH 075/146] test: hold how a /Prev to a real is judged, and a missing endstream no header stops at Two behaviors this batch set had no test that fails when they regress. The rule on trailer values follows a cross-reference stream's /Prev, or a table's, to judge what it leads to, and says both faults of a reference to a real; a hybrid file's stream's own /Prev it judges as written. Only the latter had a test, whose message both forms contain. Each now has its own, read in full. Where a guard hid a stream's /Length and its data spells no header, a missing endstream stays the file's, the search stopped at the next object the index places or at the end of the file. Every test went through an endobj of the object's own, or a header the data spells; dropping the header's part of the condition left the suite green. Refs #224, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 28 ++++--- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 80 +++++++++++++++++++ 2 files changed, 95 insertions(+), 13 deletions(-) diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index e41dadcd..60397c68 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -427,9 +427,9 @@ public static byte[] HybridNamingItsStream(string xrefStm, int padding = 0) /// A file of two classic tables: the first indexes the catalog, the page tree and the page; the second, an update, /// indexes object 4 and object 9, which holds the first table's offset, and names that table through /// , in which {offset} stands for the offset. Object 9's value comes after - /// spaces. + /// spaces; with , it is written as a real with no fractional part. /// - public static byte[] UpdateNamingItsPrevious(string prev, int padding = 0) + public static byte[] UpdateNamingItsPrevious(string prev, int padding = 0, bool real = false) { using var file = new Writer("1.4"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); @@ -440,7 +440,7 @@ public static byte[] UpdateNamingItsPrevious(string prev, int padding = 0) CultureInfo.InvariantCulture, $"xref\n0 4\n0000000000 65535 f\r\n{file.OffsetOf(1):D10} 00000 n\r\n{file.OffsetOf(2):D10} 00000 n\r\n{file.OffsetOf(3):D10} 00000 n\r\ntrailer\n<< /Size 4 /Root 1 0 R >>\nstartxref\n{first}\n%%EOF\n")); file.Object(4, "(an update)"); - file.Object(9, new string(' ', padding) + first.ToString(CultureInfo.InvariantCulture)); + file.Object(9, new string(' ', padding) + first.ToString(CultureInfo.InvariantCulture) + (real ? ".0" : string.Empty)); var second = file.Position; file.Text(string.Create( CultureInfo.InvariantCulture, @@ -738,12 +738,13 @@ public static byte[] UpdatePlacingAPaddedLengthChain(int objects, bool ownRow) } /// - /// A hybrid update over a classic table that indexes the catalog, the page tree and the page. The update writes object 9, - /// which holds 123 after spaces, and object 10, which cross-reference stream 7 indexes; the - /// stream's own /Prev is , which the reader does not follow. The update's table indexes object - /// 9, and its trailer names the older table through /Prev and the stream through /XRefStm. + /// A hybrid update over a classic table that indexes the catalog, the page tree and the page. The update writes object + /// 9, which holds after spaces, and object 10, which cross-reference + /// stream 7 indexes; the stream's own /Prev is , which the reader does not follow. + /// The update's table indexes object 9, and its trailer names the older table through /Prev and the stream + /// through /XRefStm. /// - public static byte[] HybridWithStreamPrev(string streamPrev, int padding = 3000) + public static byte[] HybridWithStreamPrev(string streamPrev, int padding = 3000, string nine = "123") { using var file = new Writer("1.5"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); @@ -753,7 +754,7 @@ public static byte[] HybridWithStreamPrev(string streamPrev, int padding = 3000) file.Text(string.Create( CultureInfo.InvariantCulture, $"xref\n0 4\n0000000000 65535 f\r\n{file.OffsetOf(1):D10} 00000 n\r\n{file.OffsetOf(2):D10} 00000 n\r\n{file.OffsetOf(3):D10} 00000 n\r\ntrailer\n<< /Size 4 /Root 1 0 R >>\nstartxref\n{older}\n%%EOF\n")); - file.Object(9, new string(' ', padding) + "123"); + file.Object(9, new string(' ', padding) + nine); file.Object(10, "(ten)"); var stream = file.Position; var data = Encode([(1, file.OffsetOf(10), 0)], Rows.Raw); @@ -799,10 +800,11 @@ public static byte[] HybridNamingATable(string key, string value, int padding = /// /// A classic table updated by a cross-reference stream. The table indexes the catalog, the page tree, the page and /// object 12, the highest number; cross-reference stream 10, the update, indexes the first three with object 4, a page - /// the tree does not list, and object 9, which holds the table's offset after spaces, and - /// names the table through /Prev 9 0 R. The table and the stream both give /Size 13. + /// the tree does not list, and object 9, which holds the table's offset after spaces — + /// written as a real with no fractional part with —, and names the table through + /// /Prev 9 0 R. The table and the stream both give /Size 13. /// - public static byte[] StreamUpdateHidingItsHighestNumber(int padding) + public static byte[] StreamUpdateHidingItsHighestNumber(int padding, bool real = false) { using var file = new Writer("1.5"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); @@ -814,7 +816,7 @@ public static byte[] StreamUpdateHidingItsHighestNumber(int padding) CultureInfo.InvariantCulture, $"xref\n0 4\n0000000000 65535 f\r\n{file.OffsetOf(1):D10} 00000 n\r\n{file.OffsetOf(2):D10} 00000 n\r\n{file.OffsetOf(3):D10} 00000 n\r\n12 1\n{file.OffsetOf(12):D10} 00000 n\r\ntrailer\n<< /Size 13 /Root 1 0 R >>\nstartxref\n{first}\n%%EOF\n")); file.Object(4, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); - file.Object(9, new string(' ', padding) + first.ToString(CultureInfo.InvariantCulture)); + file.Object(9, new string(' ', padding) + first.ToString(CultureInfo.InvariantCulture) + (real ? ".0" : string.Empty)); var stream = file.Position; var data = Encode( [(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), (1, file.OffsetOf(4), 0), (1, file.OffsetOf(9), 0), (1, stream, 0)], diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 819e1b51..35a9694c 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -228,6 +228,64 @@ public void Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s(stri new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.ObjectEndObjMissing).Should().BeFalse(); } + [Theory] + [InlineData("cut", "next object")] + [InlineData("left out", "next object")] + [InlineData("cut", "end of file")] + [InlineData("left out", "end of file")] + public void A_missing_endstream_no_header_stops_the_search_at_is_the_file_s_where_a_guard_hid_the_length(string guard, string stop) + { + // Object 4, whose /Length a guard hid, lost its endstream and its endobj, and its data spells no header: the search + // stops at the next object the index places, or at the end of the file. Nothing the guard hides would say where the + // data ends otherwise: the missing endstream is the file's, as read whole (#224). + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + }; + + if (stop == "end of file") + { + WriteOthers(); + } + + rows[4] = file.Position; + file.Raw("4 0 obj\n<< /Length 10 0 R >>\nstream\nABC\n"); + + if (stop == "next object") + { + WriteOthers(); + } + + file.Table(11, rows, "/Size 11 /Root 1 0 R"); + var limits = guard == "cut" + ? PdfReaderLimits.Default with { MaxObjectLength = 1024 } + : PdfReaderLimits.Default with { MaxIndexEntries = 8 }; + using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = limits }); + + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + stream.Data.CutByGuard.Should().BeFalse(); + document.Reader.TryGetStreamLengthFault(4, out var fault).Should().BeTrue(); + fault.Form.Should().Be(guard == "cut" ? StreamLengthForm.ObjectCut : StreamLengthForm.ObjectUnindexed); + var code = stop == "next object" ? PdfDiagnosticCodes.StreamLengthInvalid : PdfDiagnosticCodes.StreamTruncated; + document.Diagnostics.Should().ContainSingle(entry => entry.Code == code).Which.Message.Should().StartWith( + $"The stream's /Length names object 10 0, which one of the reader's limits {(guard == "cut" ? "cut" : "left out of the index")}, " + + $"and no endstream follows its data before the {(stop == "next object" ? "next object" : "end of the file")}"); + + if (stop == "next object") + { + stream.GetRawBytes().ToArray().Should().Equal("ABC\n"u8.ToArray()); + } + + void WriteOthers() + { + rows[6] = file.Object(6, "(six)"); + rows[10] = file.Object(10, (guard == "cut" ? new string(' ', 2000) : string.Empty) + "3"); + } + } + [Theory] [InlineData("cut")] [InlineData("left out")] @@ -1229,6 +1287,28 @@ public void A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_throug judged.Diagnostics.Should().BeEmpty(); } + [Theory] + [InlineData("stream", "writes its /Prev as the reference 9 0 R, to a real number: ISO 32000-1 makes it direct (7.5.8.2), and Table 17 of ISO 32000-1 an integer.")] + [InlineData("table", "writes its /Prev as the reference 9 0 R, to a real number: ISO 32000-1 makes it direct (Table 15), and Table 15 of ISO 32000-1 an integer.")] + [InlineData("hybrid stream", "writes its /Prev as the reference 9 0 R, where ISO 32000-1 makes it direct (7.5.8.2).")] + public void A_prev_written_as_a_reference_to_a_real_is_judged_through_it_only_where_the_reader_follows_it(string section, string expected) + { + // Object 9, which a /Prev names, holds a real with no fractional part. The reader follows a cross-reference stream's + // /Prev and a table's: the rule says both faults of each. A hybrid file's stream's own /Prev it does not follow, and + // the rule judges it as written, a reference (#263). + var file = section switch + { + "stream" => ChainFiles.StreamUpdateHidingItsHighestNumber(padding: 0, real: true), + "table" => ChainFiles.UpdateNamingItsPrevious("9 0 R", real: true), + _ => ChainFiles.HybridWithStreamPrev("9 0 R", padding: 0, nine: "4.0"), + }; + using var document = PdfDocument.Open(file); + var trailers = new ValidationProfile("trailers", 1, [new TrailerValueWrongRule()]); + + new PdfValidator(new PdfValidatorOptions { Profile = trailers }).Validate(document).Findings.Select(finding => finding.Message) + .Should().ContainSingle().Which.Should().EndWith(expected); + } + [Theory] [InlineData("/Filter /FlateDecode /DecodeParms << /Columns 5 0 R >>", true, false)] [InlineData("/Filter /FlateDecode /DecodeParms << /EarlyChange 5 0 R >>", true, false)] From 63faa18df1a34004fce6244190eeedd28a4756e7 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 19:52:31 +0000 Subject: [PATCH 076/146] docs: say where a guard hides a missing endstream, and what the chain reads through The pages that said a missing endstream behind a /Length a guard hid is always the file's now say where it is not: at a header the data spells, or at an endstream the object's endobj does not follow, the data up to there is the guard's. They also say that a whole reference a cut left leads on as read whole, and that a table an /XRefStm names keeps its own links unread. The threat model says how a deferred length's data whose end a guard hides is decoded, and what an R at a window's edge is asked of the file, and cites each test this round added. The rule on trailer values says which values it judges as written. Its public summary no longer claims the reader reads every value it judges. Refs #224, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- ...lid-pdf-is-readable-and-the-readers-guards-are.md | 4 +++- docs/architecture.md | 7 +++++-- docs/milestones/M02.md | 4 +++- docs/threat-model.md | 12 ++++++------ docs/website/docs/concepts/diagnostics.md | 6 ++++-- docs/website/docs/concepts/lazy-reading.md | 3 ++- docs/website/docs/concepts/reader-limits.md | 7 +++++-- docs/website/docs/reference/diagnostics.md | 5 +++-- docs/website/docs/reference/reader-limits.md | 8 +++++++- docs/website/docs/reference/validation-rules.md | 4 ++-- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 3 ++- src/AdCodicem.Pdf/IO/StreamLengthForm.cs | 7 +++++-- src/AdCodicem.Pdf/Validation/PdfValidationRuleIds.cs | 6 ++++-- tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 4 ++-- 14 files changed, 53 insertions(+), 27 deletions(-) diff --git a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md index 2d0984c2..4a967439 100644 --- a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md +++ b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md @@ -114,7 +114,9 @@ and no guard's, unless that `/Length` leads to an object `MaxObjectLength` cut ( to an object a guard cut or left out of the index — a stream's `/Length`, `/Filter` or `/DecodeParms`, a value of a filter's parameters, a cross-reference stream's `/W`, `/Index` or `/Size`, a trailer's `/Prev` or `/XRefStm` — is the guard's, not the file's: an integer the cut parted, `1` of `15`, is only a place to look for `endstream` at, never a -declared length, though a stream that lacks its `endstream` is still the file's fault; a chain of filters ends there, +declared length, though a stream that lacks its `endstream` is still the file's fault, unless a header its data spells, +or an `endstream` its object's `endobj` does not follow, stops the search, the data up to it then the guard's; a whole +reference the cut left leads on, as read whole; a chain of filters ends there, its data left encoded; and a cross-reference section whose values lead to an object `MaxObjectLength` cut ends the chain, nothing rebuilt, the older sections and their trailers the guard's to hide, and the guard reached once the chain is read. So does a cross-reference stream's `/Length` the chain could not read that leads to such an object, wherever diff --git a/docs/architecture.md b/docs/architecture.md index 41969915..86cac3fd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -113,7 +113,8 @@ separate types, separate severity scales, and vocabularies that never share a co trailers, or, once the searches of a document have read four times the file, on which streams were searched first ([#138], M23). A stream whose `/Length` a guard cut or left out of the index may be valid, its length only hidden: when a header its data spells — an embedded file's does — stops its search, no `endobj` of its own before - it, the data up to the header is the guard's, cut by it, and nothing is said of the file ([#224]). With no + it, or an `endstream` it spells, which its object's `endobj` does not follow, ends it, the data up to there is the + guard's, cut by it, and nothing is said of the file ([#224]). With no `endstream` before the stop, the object's `endobj` ends the data where it follows the declared length as an `endstream` would, or where it is the last one, a whole keyword — no regular character, solidus or percent sign right before it, which would make it part of a token, a name or a comment that @@ -132,7 +133,9 @@ separate types, separate severity scales, and vocabularies that never share a co that leads to an object a guard cut or left out of the index — a stream's `/Length`, `/Filter` or `/DecodeParms`, a cross-reference stream's `/W`, a trailer's `/Prev` or `/XRefStm` — is not judged as one the file gave. The stream's data is found by its `endstream`, an integer the cut left only a place to look for - one at, and a missing `endstream` is still the file's fault; its chain of filters ends there, with the data + one at, and a missing `endstream` is still the file's fault but where the search stops at a header the data + spells or an `endstream` its `endobj` does not follow (item 4); a reference the cut left whole is read through, + as a load reads it; its chain of filters ends there, with the data left encoded; and the chain of cross-reference sections ends at that section, with nothing rebuilt and the older sections and their trailers the guard's to hide. An object `MaxObjectLength` cut there is loaded once the chain is read, which reaches the guard — thrown at opening under `ThrowOnLimit`. A cross-reference diff --git a/docs/milestones/M02.md b/docs/milestones/M02.md index 793dbab6..c7ad5dfc 100644 --- a/docs/milestones/M02.md +++ b/docs/milestones/M02.md @@ -112,7 +112,9 @@ fault and no guard's, unless that `/Length` leads to an object a guard cut ([#22 array and dictionary an object, or a trailer, left open, and the object lacks its `endobj` ([#230]). A stream's `/Length`, `/Filter` or `/DecodeParms`, or a filter's parameter, that leads to an object a guard cut or left out of the index is the guard's, never judged as a value the file gave: the length is not judged, the data is found by -its `endstream`, and one missing is still the file's fault; the filters end there, the rest of them left undone, +its `endstream`, and one missing is still the file's fault, unless a header the data spells, or an `endstream` its +object's `endobj` does not follow, stops the search, the data up to it then the guard's; the filters end there, the +rest of them left undone, `HasImageFilter` seeing no image past it, and an object stream so left is hidden as one a guard cut, `xref.checked-in-part`'s rather than `xref.object-stream-broken`'s. A cross-reference stream whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` — or whose `/Length`, when no `endstream` ends its data — leads to an object diff --git a/docs/threat-model.md b/docs/threat-model.md index 93b8ab78..270a8189 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -248,7 +248,7 @@ Known gaps: | The lexer looping, or reading past its buffer | Every token but the end of input consumes a byte. Look-aheads check the length, and a string that never closes ends at the buffer's end, and is told from a closed one by its token's length | `PropertyTests.The_lexer_terminates_on_any_bytes_and_stays_inside_the_buffer`; `LexerTests.Never_loops_on_a_stray_delimiter`, `Recovers_from_an_unterminated_literal_string`; `ParserTests.Reports_nothing_of_a_construct_closed_at_the_end_of_the_data` | Property, over buffers of about 100 bytes | | Nesting that exhausts the stack | `PdfObjectParser.MaxDepth` (128) and a stack probe at each container. Past either, `SkipContainer` counts brackets without recursion, and `syntax.depth-exceeded` is reported | `ParserTests.Refuses_to_follow_nesting_deeper_than_its_limit`; `HostileInputTests.Survives_deeply_nested_containers_inside_an_object`, `A_file_nesting_objects_deeper_than_the_stack_allows_reads_them_as_null_without_crashing` | No | | A token or an object of any length, or one that never closes | The window grows ×8 from 8 KB up to `MaxObjectLength`, and the parser is told whether its window ends the data: only then is what it leaves open reported, once, `syntax.truncated-object` where the innermost construct opens, and a window that ends where the file does cuts nothing however long it is. Past the bound, the object is kept as far as it was read and `limit.object` is reported, beside what the parse met before the cut; nothing is reported of the token the cut reaches, nor of what follows once a look-ahead met it (ADR 34). An array or a dictionary an `endobj` finds open ends there, rather than take the objects after it, and one that lost its own `endobj` too ends at the next object's header (below). The parses of one object add up to about 1.3 times the last window | `HostileInputTests.Survives_a_string_longer_than_the_largest_window`, `A_string_that_never_closes_takes_the_end_of_the_file_and_is_reported_once_where_it_opens`; `ReaderLimitsTests.An_object_is_read_no_further_than_its_bound`, `An_object_cut_by_its_bound_is_not_reported_as_damage`, `An_object_its_bound_cuts_anywhere_in_its_last_tokens_reports_the_bound_alone`, `What_an_object_its_bound_cuts_met_before_the_cut_is_still_reported`, `A_fault_after_a_stream_whose_length_the_file_cannot_hold_is_still_reported_beside_the_bound`, `An_object_its_bound_cuts_after_more_faults_than_the_diagnostics_hold_still_reports_the_bound`, `A_number_its_bound_cuts_is_no_number_out_of_range`, `An_object_as_long_as_its_bound_that_ends_the_file_is_read_whole`; `WindowEdgeTests.An_anomaly_inside_an_object_longer_than_the_window_is_reported_once`; `PropertyTests.What_a_window_reports_of_its_syntax_the_whole_data_reports_too_in_the_same_order`, `A_buffer_that_ends_at_a_window_s_edge_reports_nothing_of_the_end_of_the_data`; `SyntaxFaultTests.A_construct_the_file_ends_inside_is_reported_where_it_opens`, `A_container_an_endobj_ends_takes_nothing_of_the_objects_after_it`, `A_fault_of_an_object_read_through_a_window_the_file_goes_on_past_is_reported`, `A_rebuild_reports_no_fault_where_its_window_cuts_a_trailer`, `A_trailer_a_source_cut_short_ends_inside_is_reported_as_the_end_of_the_data`; `FileRuleTests.A_trailer_whose_closing_brackets_end_its_table_s_window_is_judged_by_what_they_close` | Growth yes, the bound no; the window's edge by property | -| An object that lost both its closing delimiter and its `endobj` taking in the next object, up to that object's `endobj`, its own missing `endobj` unreported; a trailer that lost its `>>` taking the next update's entries; a header the window's edge cut short reported as the file's stray keys | Inside an array or a dictionary, where an element, a key or a value is expected, the next object's header ends every container left open: `N G obj`, digits alone, `N` in 1..`int.MaxValue` and `G` in 0..65,535, white space alone between them, and `obj` a whole keyword the window's edge does not reach — the header the search for a lost `endstream` stops at (`HeaderText`). A header the edge cuts short, `6 0` or `6 0 ob`, is the edge's, as a reference it cuts is: where no larger window follows — at `MaxObjectLength`, in the rebuild's trailer scan —, its numbers are not reported as keys that are not names, unless a whole token already rules a header out — a number 0, a generation past 65,535, a member, which no header follows —; `6 0 ob`, a keyword `obj` starts after two integers, which no reference can be, ends the containers as a header does, its number given to no key and nothing reported, while `6 0` alone, what the edge may have left of a reference, is kept as far as it was read, as any value the edge cuts. It is reported once, at the innermost container, as `syntax.truncated-object`, "An object header ended the object inside …", or, in a trailer or a value read alone, where no object is open, "An object header came inside …", and the object lacks its `endobj`. Containers `SkipContainer` passes over past `MaxDepth` end there too, and a trailer is read so as well. An object stream's member, which no header follows, and a value alone, which is no container, do not apply it: in `4 0 obj 5 0 obj`, object 4 holds the integer 5. Its cost, accepted: a closed container holding text that reads as a header ends there, and its `endobj` is judged missing; none of the 344,361 objects of the corpus holds one ([#230]). An `endobj` the window of `MaxObjectLength` ends exactly with is asked of the file, which says whether a regular character goes on past it: an object exactly as long as the guard reads whole, loaded or read by the chain, wherever its `endobj` comes — after its value, where a container it left open is, or as all an empty object holds ([#174]). A reference alone that the window's edge cuts after its generation leaves what follows it the edge's: the object is not said to lack its `endobj` at the guard's cut ([#224]) | `SyntaxFaultTests.An_object_that_lost_its_closing_delimiter_and_its_endobj_ends_at_the_next_object_s_header`, `The_next_object_s_header_ends_every_container_left_open_and_is_reported_at_the_innermost`, `Text_that_is_no_object_header_ends_no_container`, `A_header_inside_a_closed_container_ends_it_there_and_its_endobj_is_judged_missing`, `An_object_stream_member_holding_a_header_reads_it_as_tokens`, `Nesting_deeper_than_the_reader_follows_ends_at_the_next_object_s_header_too`, `A_trailer_that_lost_its_closing_delimiter_takes_nothing_of_the_update_after_it`, `A_header_the_guard_s_cut_leaves_undecided_is_the_guard_s`; `ParserTests.Ends_the_containers_an_object_left_open_at_the_next_object_s_header_and_leaves_it_to_be_read`, `Takes_an_object_header_a_window_s_edge_reaches_for_none_until_a_larger_window_shows_obj_whole`, `A_header_where_a_key_or_a_value_is_expected_ends_the_dictionary`, `A_value_alone_that_is_a_header_s_number_is_an_integer`, `Tokens_a_window_s_edge_leaves_undecided_as_a_header_are_no_stray_keys`; `WindowEdgeTests.An_object_that_lost_its_end_ends_at_the_next_header_wherever_the_window_edge_cuts_it`; `SyntaxFaultTests.A_header_the_guard_s_cut_leaves_undecided_gives_the_key_before_it_no_value`, `A_value_no_header_can_begin_with_keeps_its_place_at_the_guard_s_cut`, `An_object_a_guard_cuts_at_a_header_it_leaves_undecided_is_not_said_to_lack_its_endobj`, `Tokens_no_header_begins_with_are_reported_at_the_guard_s_cut_as_past_it`, `An_object_exactly_as_long_as_MaxObjectLength_reads_whole`, `An_object_the_chain_reads_exactly_as_long_as_MaxObjectLength_reads_whole`, `An_object_whose_endobj_ends_its_containers_at_MaxObjectLength_reads_as_under_a_larger_bound`, `An_object_a_guard_cuts_inside_a_reference_it_holds_alone_is_not_said_to_lack_its_endobj`; `ParserTests.Ends_a_container_skipped_for_its_depth_at_the_next_object_s_header`, `Reports_the_keys_of_a_member_a_window_s_edge_leaves_as_a_header_none_can_be` | Plausibly | +| An object that lost both its closing delimiter and its `endobj` taking in the next object, up to that object's `endobj`, its own missing `endobj` unreported; a trailer that lost its `>>` taking the next update's entries; a header the window's edge cut short reported as the file's stray keys | Inside an array or a dictionary, where an element, a key or a value is expected, the next object's header ends every container left open: `N G obj`, digits alone, `N` in 1..`int.MaxValue` and `G` in 0..65,535, white space alone between them, and `obj` a whole keyword the window's edge does not reach — the header the search for a lost `endstream` stops at (`HeaderText`). A header the edge cuts short, `6 0` or `6 0 ob`, is the edge's, as a reference it cuts is: where no larger window follows — at `MaxObjectLength`, in the rebuild's trailer scan —, its numbers are not reported as keys that are not names, unless a whole token already rules a header out — a number 0, a generation past 65,535, a member, which no header follows —; `6 0 ob`, a keyword `obj` starts after two integers, which no reference can be, ends the containers as a header does, its number given to no key and nothing reported, while `6 0` alone, what the edge may have left of a reference, is kept as far as it was read, as any value the edge cuts. It is reported once, at the innermost container, as `syntax.truncated-object`, "An object header ended the object inside …", or, in a trailer or a value read alone, where no object is open, "An object header came inside …", and the object lacks its `endobj`. Containers `SkipContainer` passes over past `MaxDepth` end there too, and a trailer is read so as well. An object stream's member, which no header follows, and a value alone, which is no container, do not apply it: in `4 0 obj 5 0 obj`, object 4 holds the integer 5. Its cost, accepted: a closed container holding text that reads as a header ends there, and its `endobj` is judged missing; none of the 344,361 objects of the corpus holds one ([#230]). An `endobj` the window of `MaxObjectLength` ends exactly with is asked of the file, which says whether a regular character goes on past it: an object exactly as long as the guard reads whole, loaded or read by the chain, wherever its `endobj` comes — after its value, where a container it left open is, or as all an empty object holds ([#174]). A reference alone that the window's edge cuts after its generation leaves what follows it the edge's: the object is not said to lack its `endobj` at the guard's cut ([#224]). An `R` the edge reaches is asked of the file too: where a regular byte goes on past it, `6 0 Rx` or `6 0 RG`, the edge cut what may be a reference, and a `/Filter` or `/Length` read through the object goes no further than under a bound one byte smaller, rather than follow a reference the file never wrote past the guard's cut ([#224], [#263]) | `SyntaxFaultTests.An_object_that_lost_its_closing_delimiter_and_its_endobj_ends_at_the_next_object_s_header`, `The_next_object_s_header_ends_every_container_left_open_and_is_reported_at_the_innermost`, `Text_that_is_no_object_header_ends_no_container`, `A_header_inside_a_closed_container_ends_it_there_and_its_endobj_is_judged_missing`, `An_object_stream_member_holding_a_header_reads_it_as_tokens`, `Nesting_deeper_than_the_reader_follows_ends_at_the_next_object_s_header_too`, `A_trailer_that_lost_its_closing_delimiter_takes_nothing_of_the_update_after_it`, `A_header_the_guard_s_cut_leaves_undecided_is_the_guard_s`; `ParserTests.Ends_the_containers_an_object_left_open_at_the_next_object_s_header_and_leaves_it_to_be_read`, `Takes_an_object_header_a_window_s_edge_reaches_for_none_until_a_larger_window_shows_obj_whole`, `A_header_where_a_key_or_a_value_is_expected_ends_the_dictionary`, `A_value_alone_that_is_a_header_s_number_is_an_integer`, `Tokens_a_window_s_edge_leaves_undecided_as_a_header_are_no_stray_keys`; `WindowEdgeTests.An_object_that_lost_its_end_ends_at_the_next_header_wherever_the_window_edge_cuts_it`; `SyntaxFaultTests.A_header_the_guard_s_cut_leaves_undecided_gives_the_key_before_it_no_value`, `A_value_no_header_can_begin_with_keeps_its_place_at_the_guard_s_cut`, `An_object_a_guard_cuts_at_a_header_it_leaves_undecided_is_not_said_to_lack_its_endobj`, `Tokens_no_header_begins_with_are_reported_at_the_guard_s_cut_as_past_it`, `An_object_exactly_as_long_as_MaxObjectLength_reads_whole`, `An_object_the_chain_reads_exactly_as_long_as_MaxObjectLength_reads_whole`, `An_object_whose_endobj_ends_its_containers_at_MaxObjectLength_reads_as_under_a_larger_bound`, `An_object_a_guard_cuts_inside_a_reference_it_holds_alone_is_not_said_to_lack_its_endobj`; `GuardCutValueTests.An_R_a_window_s_edge_reaches_makes_a_reference_only_where_the_file_ends_it_there`; `ParserTests.Ends_a_container_skipped_for_its_depth_at_the_next_object_s_header`, `Reports_the_keys_of_a_member_a_window_s_edge_leaves_as_a_header_none_can_be` | Plausibly | | Look-ahead that makes parsing superlinear | Look-ahead is a constant number of tokens, so each token is lexed three times at most | `ParserTests.Distinguishes_a_reference_from_two_integers`, `Backtracks_when_two_integers_are_not_followed_by_R` | Property | | A number past a `long` or a `double` | The number parser never wraps: an integer past a `long` reads as the real nearest to it, and a number past a double's range as an infinity, which the object parser reads as null under `syntax.number-out-of-range`, quoting the number only in a report the diagnostics keep. A real reads as the double nearest to its decimal. The shape — signs, then digits around at most one period — is checked before the framework's parse, which would read `NaN` and `Infinity`; each digit is read twice at most, and nothing is allocated | `LexerTests.Reads_an_integer_too_large_for_a_long_as_a_real`, `Reads_a_number_past_the_largest_double_as_an_infinity_its_reader_can_tell`, `Parsing_a_number_allocates_nothing_however_long_it_is`; `PropertyTests.The_integer_parser_reads_every_long_and_never_wraps_past_one`, `The_real_parser_reads_the_double_nearest_to_the_decimal`; `ParserTests.Reads_a_number_past_the_largest_real_as_null_and_reports_it`, `Reports_no_number_its_double_holds_however_it_rounds`; `HostileInputTests.Reads_a_number_fifteen_million_digits_long_in_time`, `Quotes_no_number_beyond_what_a_real_can_hold_once_the_diagnostics_are_full` | Property | | An object number or a generation overflowing its cast | The parser builds a header only from a number in 1..`int.MaxValue`, and a reference only from one in 0..`int.MaxValue` — object 0 heads the free list and reads as null —, each with a generation in 0..65,535 (`PdfObjectId.MaxNumber` and `MaxGeneration`, with their reasons), checked before the cast. The index and object streams check theirs the same way, before narrowing: see *Indexing* and *Resolving objects and object streams* | `ParserTests.Refuses_an_object_header_whose_number_or_generation_is_out_of_range`, `Makes_no_reference_of_numbers_an_object_identifier_cannot_hold`; `HostileInputTests.Serves_no_object_under_a_number_past_a_long_that_once_wrapped_to_its_number` | No | @@ -281,9 +281,9 @@ Known gaps: | A missing section tried at many places | Candidates within 512 bytes either side, nearest first, and 32 attempts at most per document (`MaxRelocationCandidates`); a section a guard cut is where it was named, and is tried nowhere else | `CrossReferenceChainTests.Reads_a_section_that_prev_names_a_few_bytes_off`, `Sections_named_a_few_bytes_off_are_looked_for_at_thirty_two_places_a_document_at_most`; `HostileInputTests.Opening_a_chain_of_hybrid_sections_whose_streams_never_close_reads_a_bounded_amount` | Yes | | A classic table of any length | Its window grows ×4 from 64 KB up to `MaxXRefSectionLength`, and the rows read are kept. A table the guard cuts, or whose rows run past a full index, keeps its trailer, found past the rows left unread by a search a megabyte at a time (`ReadTrailerPastRows`): comments are skipped, and anything else no row holds is the section's fault, reported, the search going on to the `trailer` keyword, so that its `/Encrypt` still refuses the document. The search starts at the start of the line the cut fell in, outside any comment and past the table's own keyword; it reads each megabyte once, every byte of it looked at before the next is read, however many comments or stray bytes it holds, and is not made for a place a section may have been moved to, a guess. The chain ends there, since an older section's rows would stand for the objects the unread rows define. What those rows place reads as null until the guard is raised | `ReaderLimitsTests.A_table_whose_keywords_the_bound_cuts_reports_the_bound`, `A_table_its_length_bound_cuts_keeps_its_trailer_and_the_encryption_it_names`, `A_table_its_length_bound_cuts_ends_the_chain_though_its_trailer_names_an_older_section`, `What_ends_the_rows_a_length_bound_cut_is_judged_as_it_is_where_none_did`, `A_table_a_length_bound_cuts_that_runs_to_the_end_of_the_file_has_no_trailer`, `A_table_its_length_bound_cuts_keeps_the_encryption_its_trailer_names_past_what_no_row_holds`, `A_table_cut_before_its_keyword_ends_keeps_its_trailer`, `Rows_past_a_full_index_are_not_read_and_the_trailer_past_them_is`, `A_cut_in_a_comment_among_the_rows_searches_for_the_trailer_from_the_comment_s_start`, `The_trailer_past_rows_left_unread_is_found_wherever_the_search_s_window_ends`, `A_trailer_past_rows_left_unread_that_its_own_bound_cuts_keeps_what_precedes_the_cut`; `DocumentReaderTests.The_search_for_the_trailer_past_rows_left_unread_reads_the_file_once`; `CrossReferenceChainTests.Reads_an_object_a_length_bound_cut_out_of_its_table_as_null_until_the_guard_is_lifted`, `A_section_found_near_where_prev_names_it_is_not_searched_past_the_rows_a_guard_left_unread`; `WindowEdgeTests.A_classic_table_reads_whole_wherever_its_window_edge_falls_near_its_trailer` | No: seeds stay under 120 KB | | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | -| A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s` | Yes | -| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it, and what decoding it met beside the guard dropped, a filter run over data a spelled `endstream` cut short ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut` | No | +| A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). A hybrid file's stream's own `/Prev`, which the chain does not follow, going on through the table's, is not read at all, nor are the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place: neither names a section missing nor ends the chain for a guard, and the rule on trailer values judges them as written, a reference they hold not followed ([#263]). An object the guard cut after a whole reference it holds hides nothing: the chain reads through it, as a load does, and loads it once the chain is read. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s`, `A_table_an_xrefstm_names_in_a_stream_s_place_has_its_own_links_left_unread`, `A_prev_written_as_a_reference_to_a_real_is_judged_through_it_only_where_the_reader_follows_it` | Yes | +| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it. Wherever the guard hides where the data ends — a guard on decoding kept it from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut is: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | @@ -348,12 +348,12 @@ Known gaps: | Threat | Defense | Held by | Fuzzed | |---|---|---|---| | A `/Length` that is negative, past `int.MaxValue`, not an integer or unresolvable | Only 0..`int.MaxValue` is taken as a length. Anything else sends the reader to the one search for the stream's first `endstream` (below) | `StreamLengthTests.A_length_that_cannot_be_taken_is_reported_as_the_file_wrote_it` | Partly | -| A `/Length` naming an object a guard cut or left out of the index, judged by what the cut left of it: null, reported as no length, or the `1` it parted of `15`, reported as a length the data contradicts; or the stream's missing `endstream` left unreported along with the length | Keyed on the object the reference leads to, through a chain of references too, never on the value read (`StreamLengthForm.ObjectCut`, `ObjectUnindexed`): an integer read before the cut is only a place to look for `endstream` at, never kept as a length, and the data is otherwise found by its `endstream`. Nothing is said of the length, whose fault is the guard's, reported where it was reached; a missing `endstream` is still the file's, and reported, naming the object at the end of the chain of references, the one the guard cut or left out: "The stream's /Length names object N G, which one of the reader's limits cut, and no endstream follows its data …", or "…, which one of the reader's limits left out of the index, and no endstream follows its data …". A cross-reference stream's `/Length` so cut is left alike once the chain is read, and makes its section the guard's when no `endstream` ends its data, as *Indexing* says ([#224]) | `GuardCutValueTests.A_length_a_guard_cut_is_not_judged_and_the_data_is_found_by_its_endstream`, `A_length_a_guard_cut_leaves_a_missing_endstream_the_file_s_fault`, `A_length_an_object_stream_s_guard_left_unread_is_not_judged`, `A_length_whose_reference_leads_to_an_object_a_guard_left_out_of_the_index_is_not_judged`, `A_missing_endstream_names_the_object_a_guard_cut_at_the_end_of_the_length_s_chain_and_keeps_no_length`, `A_missing_endstream_is_the_file_s_fault_when_a_guard_left_the_length_out_of_the_index`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`; `GuardCutValueTests.A_length_whose_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `A_missing_endstream_names_the_last_object_of_a_length_s_chain_a_guard_cut_or_left_out_of_the_index` | No | +| A `/Length` naming an object a guard cut or left out of the index, judged by what the cut left of it: null, reported as no length, or the `1` it parted of `15`, reported as a length the data contradicts; or the stream's missing `endstream` left unreported along with the length | Keyed on the object the reference leads to, through a chain of references too, never on the value read (`StreamLengthForm.ObjectCut`, `ObjectUnindexed`): an integer read before the cut is only a place to look for `endstream` at, never kept as a length, and the data is otherwise found by its `endstream`. Nothing is said of the length, whose fault is the guard's, reported where it was reached; a missing `endstream` is still the file's, and reported — but where the search stops at a header the data spells, or at an `endstream` its object's `endobj` does not follow (below) —, naming the object at the end of the chain of references, the one the guard cut or left out: "The stream's /Length names object N G, which one of the reader's limits cut, and no endstream follows its data …", or "…, which one of the reader's limits left out of the index, and no endstream follows its data …". A cross-reference stream's `/Length` so cut is left alike once the chain is read, and makes its section the guard's when no `endstream` ends its data, as *Indexing* says ([#224]) | `GuardCutValueTests.A_length_a_guard_cut_is_not_judged_and_the_data_is_found_by_its_endstream`, `A_length_a_guard_cut_leaves_a_missing_endstream_the_file_s_fault`, `A_length_an_object_stream_s_guard_left_unread_is_not_judged`, `A_length_whose_reference_leads_to_an_object_a_guard_left_out_of_the_index_is_not_judged`, `A_missing_endstream_names_the_object_a_guard_cut_at_the_end_of_the_length_s_chain_and_keeps_no_length`, `A_missing_endstream_is_the_file_s_fault_when_a_guard_left_the_length_out_of_the_index`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`; `GuardCutValueTests.A_length_whose_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `A_missing_endstream_names_the_last_object_of_a_length_s_chain_a_guard_cut_or_left_out_of_the_index`, `A_missing_endstream_no_header_stops_the_search_at_is_the_file_s_where_a_guard_hid_the_length` | No | | An indirect `/Length` leading back to its stream, or chained | The loading set and `MaxNestedLoads`, as above | `HostileInputTests.Survives_an_object_whose_length_refers_to_itself`, `Survives_a_chain_of_lengths_longer_than_the_stack_is_deep` | No | | A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | -| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | +| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say — that the object's `endobj` does not follow, which only the hidden length could tell from the stream's own, a reading that cannot tell where the object ends taking back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | | Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | diff --git a/docs/website/docs/concepts/diagnostics.md b/docs/website/docs/concepts/diagnostics.md index 356e5918..dd56f77d 100644 --- a/docs/website/docs/concepts/diagnostics.md +++ b/docs/website/docs/concepts/diagnostics.md @@ -71,8 +71,10 @@ the index is not judged either: its data is found the same way, and a missing `e stream may be valid, its length only hidden, and its data may spell an object header, as an embedded PDF file's does — and the `endstream` of a stream the embedded file holds, which no search can tell from the stream's own. A header the data spells that stops the search, with no `endobj` of the object's own before it, leaves the data up to it to the -guard: it is cut by the guard, as data `MaxStreamLength` cuts is, whether the object ends with `endobj` is not judged, -and nothing is said of the file. +guard, and so does an `endstream` the data spells — inside a string, say — that the object's `endobj` does not +follow: the data is marked cut by the guard, whose report the document holds, whether the object ends with `endobj` is +not judged, and nothing is said of the file. A whole reference the cut left, `6 0 R` before the window's end, is no length +it hid: it leads on to the object it names, as read whole. A cross-reference stream whose `/Length` the chain cannot read yet, which a valid file may hold, is searched as well, up to the next object the sections read so far place, or the end of the file, and its length read once the chain is. Its diff --git a/docs/website/docs/concepts/lazy-reading.md b/docs/website/docs/concepts/lazy-reading.md index eb8667ee..eea67434 100644 --- a/docs/website/docs/concepts/lazy-reading.md +++ b/docs/website/docs/concepts/lazy-reading.md @@ -63,7 +63,8 @@ sections past `MaxXRefSectionCount`, the rows past `MaxXRefSectionLength` or pas `/Length` leads to such an object, when no `endstream` ends its data before the next object or the end of the file, or the rows past an `endstream` its rows spell, which only that length could tell from the end of its data —, the sections a `/Prev` or an `/XRefStm` that leads to such an object names — the older ones, or the cross-reference stream of a hybrid -file's revision —, the numbers past `MaxIndexEntries`. That is no +file's revision, with the sections older than it, the chain ending with it —, the numbers past `MaxIndexEntries`. An +object the guard cut after a whole reference it holds hides none of them: the chain reads through it. That is no damage, and the map is not rebuilt for it. The map is rebuilt by scanning the whole file, once at most, and never while the chain is read: diff --git a/docs/website/docs/concepts/reader-limits.md b/docs/website/docs/concepts/reader-limits.md index 43fb5af3..db1ac127 100644 --- a/docs/website/docs/concepts/reader-limits.md +++ b/docs/website/docs/concepts/reader-limits.md @@ -51,8 +51,11 @@ document it came from, however long after opening. A value read through a reference to an object a guard cut, or left out of the index, is the guard's too. What the cut left of it may be the start of what the file holds — `1` of `15`, `/Flate` of `/FlateDecode` —, so it is never judged -as a value the file gave: the data of a stream whose `/Length` is so named is found by its `endstream`, and its length -is not judged, though an `endstream` the data lacks is still the file's to lack; decoding stops at a filter so named, or +as a value the file gave; a whole reference it left, though, leads on to the object it names, as read whole. The data +of a stream whose `/Length` is so named is found by its `endstream`, and its length is not judged, though an `endstream` +the data lacks is still the file's to lack — unless an object header the data spells, or an `endstream` its object's +`endobj` does not follow, stops the search: only the hidden length could tell those from the end of the data, which is +then the guard's too; decoding stops at a filter so named, or one whose parameters are, and leaves the rest of the data encoded, as past the most filters the reader runs; and a `/Prev` or an `/XRefStm` so named, or a cross-reference stream whose values lead to an object a guard cut, ends the chain as a section a guard cut does. diff --git a/docs/website/docs/reference/diagnostics.md b/docs/website/docs/reference/diagnostics.md index 5642208e..252d6cf5 100644 --- a/docs/website/docs/reference/diagnostics.md +++ b/docs/website/docs/reference/diagnostics.md @@ -186,8 +186,9 @@ judged either: the data is found by its `endstream` the same way, and a missing object the guard left out — `The stream's /Length names object 21 0, which one of the reader's limits left out of the index, and no endstream follows its data`, then where the search stopped and what was taken. For either, an object header the data spells, as an embedded PDF file's does, that stops the search — no `endobj` of the object's own before -it — leaves the data up to it to the guard: nothing is reported of the stream, whose data is cut by the guard, and -whether its object ends with `endobj` is not judged. +it —, or an `endstream` the data spells that the object's `endobj` does not follow, leaves the data up to it to the +guard: nothing is reported of the stream, whose data is cut by the guard, and whether its object ends with `endobj` is +not judged. ```text Warning stream.length-invalid at 5501: The stream declared 19954 bytes but ended after 19952. diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index 285c2971..95e41f0e 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -112,7 +112,13 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. the search for its `endstream`, and its length is not judged: an integer the cut left, which may be the first digits of a longer one, is only a place to look for `endstream` at. No `endstream` after the data of a stream whose `/Length` names an object a guard cut, or left out of the index, is still the file's fault, and reported; the report - names the object at the end of the chain of references. Decoding stops at a filter so named, or at the filter whose + names the object at the end of the chain of references. The search may stop short of it, though, at an object header + the data spells, as an embedded PDF file's does, or at an `endstream` the object's `endobj` does not follow, as a + string may spell one: the length that tells those from the end of the data is the guard's, so the data up to there + comes back cut by the guard, `CutByGuard`, whether the object ends with `endobj` is not judged, and nothing is said of + the file. A reference the cut left whole, the cut falling after `6 0 R`, hides nothing: it is read through to the + object it names, as it is read whole, while the chain is read as afterward. Decoding stops at a filter so named, or + at the filter whose parameters are: the filters before it run, and the stream comes back as they decoded it, still encoded by the rest, as past `MaxFilterChainLength`. `HasImageFilter` reads what decoding reads before the image filter, the parameters of a Flate or LZW filter that comes before it included, and says false when such a value ends the chain there; under diff --git a/docs/website/docs/reference/validation-rules.md b/docs/website/docs/reference/validation-rules.md index 131c8db9..a19f1d26 100644 --- a/docs/website/docs/reference/validation-rules.md +++ b/docs/website/docs/reference/validation-rules.md @@ -45,7 +45,7 @@ References are to ISO 32000-1:2008. | `file.trailer-malformed` | Error | A trailer is not a well-formed dictionary: the `trailer` keyword is not followed by one, or the reader read it despite syntax errors — a key that is not a name, a dictionary or a string left unclosed, a hexadecimal string holding bytes that are neither digits nor white space, a name whose number sign is no escape, a key given twice —, so an entry may have been lost with the syntax, or read otherwise than its writer meant. A cross-reference stream's dictionary is held to the same. | Each trailer of the chain, as the reader parsed it | | `file.root-invalid` | Error | The trailer's `/Root` is missing or does not lead to a document catalog (Table 15), a value that is not a reference included — unless it is a catalog written in the trailer itself, which the reader takes as it is ([#111]). The reader then looks for an object of `/Type /Catalog` among those the index holds — and rebuilds the index only when none is one —: the catalog it finds is its choice, not the file's. Not judged when the trailer could not be read at all, nor when the index was rebuilt as the document opened, nor when one of the reader's limits left the catalog out of the index as the document opened, which `xref.checked-in-part` says, or cut the object `/Root` names — through a chain of references too, and in an object stream whose cut leaves it unread, past the cut data or in a stream the guard made unreadable ([#259]), or in one whose filters end at a `/Filter`, a `/DecodeParms` or a parameter that leads to an object a guard cut or left out of the index, which leaves its data encoded ([#263]) — or the trailer a catalog written there was read from, which the reader then takes for the catalog as it read it, looking for no other, and its `limit.*` report names ([#175]) — whatever a rebuild found since —, nor a `/Root` the trailers lack where one of the reader's limits kept a trailer from being read whole — cut, or older than where the chain stopped, at the section count or at a section whose rows a limit cut or filled the index with —: it may lie there, and the reader rebuilds nothing for it. Rows a limit cut that run on to no trailer lack one: the file's, which a rebuild answers. The catalog the finding names is the one the reader took last, in the rebuild run before the first rule if there was one. | The trailer the chain gave, merged as readers merge it, and `/Root` before the reader looked for a catalog | | `file.size-wrong` | Warning | A section's `/Size` is missing, or is not one more than the highest object number it and the sections it updates use (Tables 15 and 17, 7.5.6, 7.5.8.4). A hybrid file's cross-reference stream meets one of Table 17's two readings or the other — one more than the highest number it and the sections it updates use, or the `/Size` of the table that names it —, and either is accepted. A `/Size` that leaves in-use objects numbered above it is `xref.object-past-size`'s. A `/Size` written as a reference, or as a real, is said to be. Not judged when the chain lost a section, or one of the reader's limits cut it or ended it before the sections a `/Prev` or an `/XRefStm` names ([#263]). | Each section's trailer, and the highest number each section indexes | -| `file.trailer-value-wrong` | Warning | A value a section's trailer gives the reader to read the chain is written as the specification does not allow, and read all the same, as other readers read it: a reference where it must be direct, read where a section read before it places the object; a real with no fractional part where it must be an integer, read as the integer it equals ([#182], [#215]). A cross-reference stream's `/Type`, `/Index`, `/Prev` and `/W`, the elements of `/Index` and `/W`, and its `/Filter` and `/DecodeParms` are direct (7.5.8.2); in a file that declares PDF 2.0, so are its `/Length` and the elements and entries of its `/Filter` and `/DecodeParms` (an erratum of ISO 32000-2, pdf-issues #246). A classic trailer's `/Prev` is direct (Table 15); an `/XRefStm` may be indirect (Table 19). A hybrid file's cross-reference stream's own `/Prev`, which the reader does not follow, the chain going on through the table's, is judged as the file wrote it, a reference it holds not followed. `/W`, `/Index`, `/Prev`, `/XRefStm`, a stream's `/Length` and its `/Predictor`, `/Colors`, `/BitsPerComponent`, `/Columns` and `/EarlyChange` are integers. One finding per value, which says both faults of a reference that leads to a real. A value the chain is read with that the reader could not read so is the section's fault — `xref.section-malformed`, `xref.section-not-found` —, unless it leads to an object one of the reader's limits cut, where the chain ends for the limit, a section whose rows need it left to `xref.checked-in-part` ([#263]), and `/Size` is `file.size-wrong`'s; a `/Type`, or a `/Length` read once the chain is, is reported here whether or not it could be read. | Each section's trailer, as the file wrote it, and the objects its references name, through the index the reader settled on before the first rule | +| `file.trailer-value-wrong` | Warning | A value a section's trailer gives the reader to read the chain is written as the specification does not allow, and read all the same, as other readers read it: a reference where it must be direct, read where a section read before it places the object; a real with no fractional part where it must be an integer, read as the integer it equals ([#182], [#215]). A cross-reference stream's `/Type`, `/Index`, `/Prev` and `/W`, the elements of `/Index` and `/W`, and its `/Filter` and `/DecodeParms` are direct (7.5.8.2); in a file that declares PDF 2.0, so are its `/Length` and the elements and entries of its `/Filter` and `/DecodeParms` (an erratum of ISO 32000-2, pdf-issues #246). A classic trailer's `/Prev` is direct (Table 15); an `/XRefStm` may be indirect (Table 19). A value the reader does not read is judged as the file wrote it, a reference it holds not followed, which would read what the reading never did: a hybrid file's cross-reference stream's own `/Prev`, which the chain does not follow, going on through the table's; the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place; and what of a cross-reference stream's `/Filter` and `/DecodeParms` decoding does not read — the parameters of a filter that reads none, `/Colors`, `/BitsPerComponent` and `/Columns` where no predictor is, `/EarlyChange` but for LZW, or what lies past the element that ends the chain ([#263]). `/W`, `/Index`, `/Prev`, `/XRefStm`, a stream's `/Length` and its `/Predictor`, `/Colors`, `/BitsPerComponent`, `/Columns` and `/EarlyChange` are integers. One finding per value, which says both faults of a reference that leads to a real where the reader follows it. A value the chain is read with that the reader could not read so is the section's fault — `xref.section-malformed`, `xref.section-not-found` —, unless it leads to an object one of the reader's limits cut, where the chain ends for the limit, a section whose rows need it left to `xref.checked-in-part` ([#263]), and `/Size` is `file.size-wrong`'s; a `/Type`, or a `/Length` read once the chain is, is reported here whether or not it could be read. | Each section's trailer, as the file wrote it, and the objects the references the reader follows name, through the index the reader settled on before the first rule | ### Cross-references @@ -87,7 +87,7 @@ it is checked: the exception above judges the index the reader rebuilt, and the | Rule | Severity | Meaning | Reads | |---|---|---|---| | `object.reference-missing` | Warning | An object reachable from the trailer refers to an object the file does not define — none in its index, a free entry, or object 0, which heads the free list and is never in use (7.5.4) —, which is null (7.3.10): the reader reads it so, rebuilding nothing, and what the reference was to give is lost. One finding per object that holds such references, naming the first. An object one of the reader's limits left out of the index is unknown rather than missing, and is `xref.checked-in-part`'s. An object the index holds and the reader cannot produce is `xref.entry-broken`'s or `xref.object-stream-broken`'s — but one a rebuild serves from another object stream than the chain's index places it in, or from another definition of that stream, which does not list it or holds it written as a stream, and so loses, which no finding names yet ([#276]) —, a kid of the page tree `page-tree.kid-invalid`'s — one in a node's `/Kids`, an array of kids of its own or a root written in the catalog the walk read whole, not what an array written among the kids holds, which the walk does not read —, and a trailer's `/Root` that names nothing `file.root-invalid`'s. | Every object reachable from the trailer, once each | -| `object.endobj-missing` | Warning | An object reachable from the trailer does not end with `endobj` (7.3.10): another token, or the end of the file, follows its value. The reader reads the value as far as it goes, as qpdf does while reporting it. An empty object, `2 0 obj endobj`, has its `endobj`. An object whose array or dictionary was never closed lacks it where the next object's header — `N G obj` in digits, white space alone between them, an object number from 1 to 2,147,483,647 and a generation up to 65,535 —, met where an element, a key or a value is expected, ends every container left open ([#230]); an array or a dictionary that holds such a header and is closed after it ends there all the same, its `endobj` judged missing, which none of the corpus's 344,361 objects does. What follows a stream's `endstream` is read through white space and comments however long, and the file asked for it when it lies past the 8 KB window the reader first reads through — after a stream whose data runs past that window ([#55]), or an `endstream` the window's edge comes too soon after ([#174]). A stream with no `endstream` has its `endobj` when the object's `endobj` follows the length it declared — after white space and comments, a comment only once white space comes before it —, or ends its data with nothing but white space and comments after it before the next object or the end of the file; one whose data runs on to the next object, or to the end of the file, lacks it. Where the object ends is unknown, and is not judged, after a stream whose declared length the reader kept with neither an `endstream` nor an `endobj` after it, one whose data, run on to the next object or the end of the file, holds an `endobj` that may be its object's own, and one whose end lies past what the reader reads first once the document's searches for `endstream` have read as much of the file as they may. | What follows each reachable object's value, as the reader read it | +| `object.endobj-missing` | Warning | An object reachable from the trailer does not end with `endobj` (7.3.10): another token, or the end of the file, follows its value. The reader reads the value as far as it goes, as qpdf does while reporting it. An empty object, `2 0 obj endobj`, has its `endobj`. An object whose array or dictionary was never closed lacks it where the next object's header — `N G obj` in digits, white space alone between them, an object number from 1 to 2,147,483,647 and a generation up to 65,535 —, met where an element, a key or a value is expected, ends every container left open ([#230]); an array or a dictionary that holds such a header and is closed after it ends there all the same, its `endobj` judged missing, which none of the corpus's 344,361 objects does. What follows a stream's `endstream` is read through white space and comments however long, and the file asked for it when it lies past the 8 KB window the reader first reads through — after a stream whose data runs past that window ([#55]), or an `endstream` the window's edge comes too soon after ([#174]). A stream with no `endstream` has its `endobj` when the object's `endobj` follows the length it declared — after white space and comments, a comment only once white space comes before it —, or ends its data with nothing but white space and comments after it before the next object or the end of the file; one whose data runs on to the next object, or to the end of the file, lacks it. Where the object ends is unknown, and is not judged, after a stream whose declared length the reader kept with neither an `endstream` nor an `endobj` after it, one whose data, run on to the next object or the end of the file, holds an `endobj` that may be its object's own, and one whose end lies past what the reader reads first once the document's searches for `endstream` have read as much of the file as they may. So is a stream whose `/Length` one of the reader's limits cut or left out of the index, when an object header its data spells, or an `endstream` it spells that the object's `endobj` does not follow, ends its data: only that length could tell either from the stream's own end ([#224]). | What follows each reachable object's value, as the reader read it | | `object.name-null-character` | Warning | An object reachable from the trailer holds a name with a null character, written `#00`, which a name cannot contain (7.3.5). The reader keeps the name; qpdf refuses it and reads dictionaries without the keys so named. One finding per object, naming the first. | Every name of every reachable object | | `object.key-missing` | Warning | An object reachable from the trailer lacks a key the [Arlington PDF Model](#object-shapes-from-the-arlington-pdf-model) requires of its type — absent, or `null`, which in a dictionary is the same (7.3.7): a key the model requires once the version the file declares has it, or one it requires in some versions only, such as a form XObject's `/Name` in PDF 1.0. An inheritable key is looked for up the `/Parent` chain, and a field's `/DA` in the interactive form's too; a key only an extension of ISO 32000 defines is never required. An array shorter than the elements it requires lacks each required element it does not hold, one finding per element. [One finding per type and key](#object-shapes-from-the-arlington-pdf-model), at the first object, with how many lack it. | Every object reachable from the trailer, typed along the model's links, once each | | `object.value-type-wrong` | Warning | A value is of a type the Arlington PDF Model does not allow for its key, judged by the value's class: a real, even `400.0`, where an integer belongs; a string where a name does; a stream where a dictionary does, or the reverse, which is reported on the object that holds it and not walked into. A `null` written in an array is judged like any other value; one reached through a reference is not judged. One finding per type and key. | As `object.key-missing` | diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 37596751..56a1a7c6 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -1223,7 +1223,8 @@ private bool KeepLengthFault(StreamLengthFault fault) { // A /Length read once the cross-reference chain is: whether the data ends where it should is known then (#182). One a // guard left out of the index, or cut, is unknown, and the guard's report says why; a missing endstream is still the - // file's (#224), and the only fault reported of it. + // file's (#224), and the only fault reported of it, where no header the data spells, nor an endstream its endobj + // does not follow, stops the search. if (fault.Form == StreamLengthForm.Deferred) { return false; diff --git a/src/AdCodicem.Pdf/IO/StreamLengthForm.cs b/src/AdCodicem.Pdf/IO/StreamLengthForm.cs index 7613a9eb..72151bb4 100644 --- a/src/AdCodicem.Pdf/IO/StreamLengthForm.cs +++ b/src/AdCodicem.Pdf/IO/StreamLengthForm.cs @@ -35,7 +35,9 @@ internal enum StreamLengthForm : byte /// A reference to an object one of the reader's guards left out of the index, or that a chain of references from it /// leads to: whether the file holds it is unknown. The data ends where its endstream ends it, or as a stream with /// none ends; nothing is reported of the length, whose fault is the guard's, which reported it, and a missing - /// endstream is still the file's (#224). + /// endstream is still the file's (#224) — but where the search stops at an object header the data spells, + /// or at an endstream its object's endobj does not follow, which only the length could tell from the + /// end of the data: the data up to there is then the guard's. /// ObjectUnindexed, @@ -43,7 +45,8 @@ internal enum StreamLengthForm : byte /// A reference to an object one of the reader's guards cut, or that a chain of references from it leads to: what it /// holds may lie past the cut. An integer read before the cut is only a place to look for endstream at, and /// never kept as a length; the data is otherwise taken up to its endstream. Nothing is reported of the length, - /// whose fault is the guard's, which reported it; a missing endstream is still the file's (#224). + /// whose fault is the guard's, which reported it; a missing endstream is still the file's (#224), but where + /// the search stops as says. /// ObjectCut, } diff --git a/src/AdCodicem.Pdf/Validation/PdfValidationRuleIds.cs b/src/AdCodicem.Pdf/Validation/PdfValidationRuleIds.cs index 9fa04f27..f7d74f0d 100644 --- a/src/AdCodicem.Pdf/Validation/PdfValidationRuleIds.cs +++ b/src/AdCodicem.Pdf/Validation/PdfValidationRuleIds.cs @@ -72,8 +72,10 @@ public static class PdfValidationRuleIds /// /// A value a section's trailer — a classic table's, or a cross-reference stream's dictionary — gives the reader to read - /// the chain is written as the specification does not allow, and read all the same: a reference where it must be - /// direct, a real with no fractional part where it must be an integer. . + /// the chain is written as the specification does not allow: a reference where it must be direct, a real with no + /// fractional part where it must be an integer. The reader reads such a value all the same, or does not read it at all + /// — a hybrid file's stream's own /Prev —, which is then judged as written. + /// . /// public const string FileTrailerValueWrong = "file.trailer-value-wrong"; diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 35a9694c..97f6e6af 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -292,8 +292,8 @@ void WriteOthers() public void A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header(string guard) { // Object 4, whose /Length a guard hid, lost its endstream; its own endobj, then an object no row places, a stream - // with an endstream, follow: the header that object starts with is no header the data spells, and the endobj before - // it ends the data, the missing endstream the file's (#224). + // with an endstream, follow: the endobj, which comes before that object's header, ends the data before any header + // can, and the missing endstream is the file's (#224). var file = new HandWrittenFile(); var rows = new Dictionary { From 05ba3984bbced018f7b44750a3399480a70ebf12 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 22:00:01 +0000 Subject: [PATCH 077/146] fix: read an object stream member whose reference ends at MaxObjectLength's bound whole A member window MaxObjectLength bounds ends inside the decoded data, which is in memory, so the byte past an R its edge reaches says whether the R is whole, as the file does for an object read for the chain. The member parser was given nothing to ask, so "6 0 R" ending exactly at the bound read as the integer 6, cut, reported, and thrown under ThrowOnLimit. The object stream's contents now serve as the parser's edge; "6 0 RG" stays cut, and so does a token at the end of data a guard cut, which has nothing past it to ask. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/IPdfFileEdge.cs | 16 ++++--- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 12 ++++- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 13 +++-- .../AdCodicem.Pdf.Tests/ReaderLimitsTests.cs | 48 +++++++++++++++++++ 4 files changed, 76 insertions(+), 13 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/IPdfFileEdge.cs b/src/AdCodicem.Pdf/IO/IPdfFileEdge.cs index 17758948..347627a2 100644 --- a/src/AdCodicem.Pdf/IO/IPdfFileEdge.cs +++ b/src/AdCodicem.Pdf/IO/IPdfFileEdge.cs @@ -1,20 +1,22 @@ namespace AdCodicem.Pdf.IO; /// -/// Says what the file holds right past the end of a parser's buffer, where a window's edge leaves a token undecided. +/// Says what the file — or an object stream's decoded data — holds right past the end of a parser's buffer, where a window's +/// edge leaves a token undecided. /// /// -/// A parser whose window does not end the data asks it of an endobj its window's edge reaches, which may be the start -/// of a longer keyword: an object a window ends exactly with — the last one -/// allows, say — reads whole, as under a larger window (#174). It is -/// all a parser asks of the file when it reads an object for the chain without loading it, which reads no stream's data. +/// A parser whose window does not end the data asks it of an endobj or an R its window's edge reaches, which +/// may be the start of a longer keyword — endobjX, RG —: an object a window ends exactly with — the last one +/// allows, say — reads whole, as under a larger window (#174, #224). +/// It is all a parser asks of the file when it reads an object for the chain without loading it, which reads no stream's +/// data; an object stream member's parser asks it of the decoded data, which is in memory. /// internal interface IPdfFileEdge { /// /// Determines whether a token that reaches the end of the parser's buffer at ends there: - /// the file ends there, or no regular character follows. The parser asks of an endobj its window's edge reaches, - /// which may be the start of a longer keyword (#174). + /// the file ends there, or no regular character follows. The parser asks of an endobj or an R its window's + /// edge reaches, which may be the start of a longer keyword (#174, #224). /// bool TokenEndsAt(long absoluteOffset); } diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 1847a421..06541567 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -6546,7 +6546,7 @@ public void Clear() } /// The objects packed inside one object stream, and where each of them starts. - private sealed class ObjectStreamContents + private sealed class ObjectStreamContents : IPdfFileEdge { private readonly ReadOnlyMemory _data; private readonly int[] _numbers; @@ -6594,6 +6594,14 @@ private ObjectStreamContents( public int Count => _numbers.Length; + /// + /// Determines whether a token that reaches a member window's edge at byte of the + /// decoded data ends there: no regular character follows. A window MaxObjectLength bounds ends inside the data; past + /// the end of data a guard cut there is nothing to ask, and the token stays undecided. + /// + bool IPdfFileEdge.TokenEndsAt(long absoluteOffset) => + absoluteOffset < _data.Length && !PdfCharacters.IsRegular(_data.Span[(int)absoluteOffset]); + /// /// Gets what keeping the contents costs, in bytes: the decoded data, the header's two tables and the lookup of its /// numbers, whether built yet or not — four bytes for each table and eight for the lookup, for each member (#160). @@ -6782,7 +6790,7 @@ private ObjectStreamContents( var window = ObjectStreamMemberWindow.Of(_offsets, index, _first, _data.Length, _cutByGuard, maxLength); var parser = PdfObjectParser.ForObjectStreamMember( - _data[..window.End], _number, _position, expectedNumber, source, parsing, window.EndsData, window.EndsAtNextMember); + _data[..window.End], _number, _position, expectedNumber, source, parsing, window.EndsData, window.EndsAtNextMember, this); parser.Position = window.Start; var value = parser.ParseObject(); diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 56a1a7c6..0242e3df 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -78,8 +78,8 @@ internal ref struct PdfObjectParser /// /// Whether a window's edge left the look-ahead of an integer undecided — "12 0" that may be what the edge left of - /// "12 0 R" —: what the parse reads after the integer is what the edge made of the reference, not what follows the - /// object (#224). + /// "12 0 R", or "12 0 R" that may be what it left of "12 0 RG" —: what the parse reads after the integer is what the + /// edge made of the reference, not what follows the object (#224). /// private bool _lookAheadCut; @@ -173,6 +173,10 @@ public PdfObjectParser( /// data —, rather than where a guard cut it: only then is what its end cuts short the data's fault. /// /// Whether ends at the next member's start, which its reports then name. + /// + /// Says what the decoded data holds past 's end, where a window's edge leaves a keyword undecided: + /// the data is in memory, so a member a window ends exactly with reads whole, as under a larger window. + /// public static PdfObjectParser ForObjectStreamMember( ReadOnlyMemory data, int streamNumber, @@ -181,9 +185,10 @@ public static PdfObjectParser ForObjectStreamMember( IPdfObjectSource source, PdfDiagnostics diagnostics, bool endsData = true, - bool endsAtNextMember = false) + bool endsAtNextMember = false, + IPdfFileEdge? dataEdge = null) { - var parser = new PdfObjectParser(data, 0, source, diagnostics, endsData: endsData); + var parser = new PdfObjectParser(data, 0, source, diagnostics, endsData: endsData, fileEdge: dataEdge); parser._member = new ObjectStreamMember(streamNumber, dataStart, objectNumber, diagnostics, endsAtNextMember); return parser; } diff --git a/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs b/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs index b9c19f8a..9b5c8407 100644 --- a/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs @@ -905,6 +905,35 @@ public void A_member_whose_value_ends_where_MaxObjectLength_ends_its_window_is_r document.Diagnostics.Should().AllSatisfy(diagnostic => diagnostic.Message.Should().StartWith("An array was closed by a dictionary end. It was met in object 4,")); } + [Fact] + public void A_member_whose_reference_ends_where_MaxObjectLength_ends_its_window_is_the_reference() + { + // Member 4's window ends where MaxObjectLength does, right after the R of "6 0 R": the decoded byte past it says the + // R is whole, as the file says of an object's, so the member is the reference the window the member would have + // without the guard reads, and nothing is reported, under ThrowOnLimit as without. + using var document = PdfDocument.Open( + MemberReachingItsBound("6 0 R"), + new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = 205 }, ThrowOnLimit = true }); + + document.GetObject(new PdfObjectId(4)).Should().BeOfType().Which.Id.Should().Be(new PdfObjectId(6)); + document.Reader.IsCutAtLimit(4).Should().BeFalse(); + document.Diagnostics.Should().BeEmpty(); + } + + [Fact] + public void A_member_whose_R_the_byte_past_MaxObjectLength_continues_is_cut() + { + // The byte past member 4's window continues the R, "RG": what the window holds may be no reference, and the member + // is one the guard cut. + using var document = PdfDocument.Open( + MemberReachingItsBound("6 0 RG"), new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = 205 } }); + + _ = document.GetObject(new PdfObjectId(4)); + + document.Reader.IsCutAtLimit(4).Should().BeTrue(); + document.Diagnostics.Select(diagnostic => diagnostic.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + } + [Fact] public void A_member_whose_array_ends_where_a_guard_cuts_the_data_is_read_whole() { @@ -3159,6 +3188,25 @@ static string Row(int type, long offset, int generation) => private static int[] Numbers(int first, int count) => [.. Enumerable.Range(first, count)]; + /// + /// Writes a file whose object stream 3 holds member 4, 200 spaces, then and 300 spaces, and + /// member 6, 42: a MaxObjectLength of 205 ends member 4's window right after the R of a reference. + /// + private static byte[] MemberReachingItsBound(string value) + { + var written = new HandWrittenFile(version: "1.5"); + var rows = new Dictionary + { + [1] = (1, written.Object(1, Catalog), 0), + [2] = (1, written.Object(2, Pages), 0), + [3] = (1, written.ObjectStream(3, [(4, new string(' ', 200) + value + new string(' ', 300)), (6, "42")]), 0), + [4] = (2, 3, 0), + [6] = (2, 3, 1), + }; + written.CrossReferenceStream(7, 8, rows, "/Root 1 0 R"); + return written.Bytes; + } + /// /// Writes a file whose chain places object 4 in object stream 10, whose dictionary runs past 128 bytes, and objects 6 and /// 7 in object stream 11, which holds 6 alone; row 9 places its object inside object 8's string, so that settling From ffa84330755a0aecbe349cf0f71fc3e2067c326d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 22:05:39 +0000 Subject: [PATCH 078/146] fix: leave an endobj an endstream lacks the file's when the next indexed object follows it Under a /Length a guard hid, an endstream its object's endobj does not follow was taken for one the data may spell, and the data up to it for the guard's. One the next object the index places follows, white space and comments between, is the stream's own: no data spells an object the index places. Its object lacks the endobj, as read whole, and its Flate data, cut short, is the file's to report. The search says so once for each stream, through the reads its budget bounds. An endstream only the table follows stays the guard's: data may spell that too. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 34 +++++--- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 5 +- src/AdCodicem.Pdf/IO/StreamEndSearch.cs | 7 ++ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 87 +++++++++++++++++++ 4 files changed, 121 insertions(+), 12 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 06541567..3662c502 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -1871,17 +1871,31 @@ private StreamEndSearch SearchEndStreamOnce(int number, long absoluteDataStart, // Past int.MaxValue bytes, no length the reader holds a stream by reaches the endstream found: only a file of more // than 2 GB can put one there, and the stream is read as one whose endstream is not found. With none, what the // object's endobj does is read once with the search, as long as the searches may read. - return dataEnd >= 0 && dataEnd - absoluteDataStart <= int.MaxValue - ? new StreamEndSearch( - (int)(dataEnd - absoluteDataStart), - stop, - ((IPdfStreamDataProvider)this).CheckEndStream(dataEnd) ?? EndObjState.Unseen) - : new StreamEndSearch(null, stop, EndObjState.Unknown, searched) - { - EndObjSeen = endObjSeen, - ObjectEnd = objectEnd, - HeaderStopped = header >= 0, + if (dataEnd >= 0 && dataEnd - absoluteDataStart <= int.MaxValue) + { + var endObj = ((IPdfStreamDataProvider)this).CheckEndStream(dataEnd) ?? EndObjState.Unseen; + return new StreamEndSearch((int)(dataEnd - absoluteDataStart), stop, endObj) + { + NextObjectFollows = endObj == EndObjState.Absent && nextObject is { } next && EndStreamLeadsTo(dataEnd, next), }; + } + + return new StreamEndSearch(null, stop, EndObjState.Unknown, searched) + { + EndObjSeen = endObjSeen, + ObjectEnd = objectEnd, + HeaderStopped = header >= 0, + }; + } + + /// + /// Determines whether the first token after the endstream that ends data at , white space + /// and comments between, starts at , as far as the searches may read. + /// + private bool EndStreamLeadsTo(long dataEnd, long next) + { + var keyword = NextSignificant(dataEnd); + return keyword >= 0 && NextSignificant(keyword + StreamEnds.EndStreamKeyword.Length) == next; } /// diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 0242e3df..fe3d0e93 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -836,8 +836,9 @@ private PdfStream ReadStream(PdfDictionary dictionary) // An endstream its object's endobj does not follow, where a guard cut the /Length or left it out of the index: one // the data spells, as a valid stream's may, ends the search as the stream's own would, its object's endobj lost — // the length that tells them apart is the guard's. The data up to it is the guard's, cut by it, the object's - // endobj unjudged, and nothing is said of the file, as at a header the data spells (#224). - if (hidden && settled.EndObj == EndObjState.Absent) + // endobj unjudged, and nothing is said of the file, as at a header the data spells (#224). One the next object + // the index places follows is the stream's own, which no data spells: its object lacks the endobj. + if (hidden && settled.EndObj == EndObjState.Absent && !settled.NextObjectFollows) { _endObjUnknown = true; return Finish(dictionary, dataStart, found, span.Length, cutByGuard: true); diff --git a/src/AdCodicem.Pdf/IO/StreamEndSearch.cs b/src/AdCodicem.Pdf/IO/StreamEndSearch.cs index 0d5debdd..42984ace 100644 --- a/src/AdCodicem.Pdf/IO/StreamEndSearch.cs +++ b/src/AdCodicem.Pdf/IO/StreamEndSearch.cs @@ -45,4 +45,11 @@ internal readonly record struct StreamEndSearch(int? Length, long? NextObject, E /// then gives, rather than the next object the index places or the end of the file. /// public bool HeaderStopped { get; init; } + + /// + /// Gets a value indicating whether the next object the index places follows the endstream found, white space and + /// comments between: no endstream the data spells can be followed by an object the index places, so the one found + /// is the stream's own, and the endobj it lacks the file's, whatever hid the length (#224). + /// + public bool NextObjectFollows { get; init; } } diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 97f6e6af..6fef7a4e 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -723,6 +723,93 @@ public void An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_ new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.ObjectEndObjMissing).Should().BeFalse(); } + [Theory] + [InlineData("cut", false)] + [InlineData("left out", false)] + [InlineData("cut", true)] + [InlineData("left out", true)] + public void An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault(string guard, bool flate) + { + // Object 4's endstream lost its endobj, and object 5, which the table places, follows it directly. No endstream the + // data spells can be followed by an object the index places, so the one found is the stream's own, whatever the + // guard hid: what the object lacks is the file's, as read whole, and so is the end of its Flate data, which lost its + // last bytes (#224). + var data = "BT /F1 12 Tf 72 712 Td (Hello) Tj ET"; + + if (flate) + { + using var output = new MemoryStream(); + + using (var zlib = new ZLibStream(output, CompressionLevel.NoCompression, leaveOpen: true)) + { + zlib.Write(Encoding.ASCII.GetBytes(data)); + } + + data = Encoding.Latin1.GetString(output.ToArray()[..^6]); + } + + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, Page), + [20] = file.Object(20, (guard == "cut" ? new string(' ', 2000) : string.Empty) + data.Length.ToString(CultureInfo.InvariantCulture)), + [4] = file.Position, + }; + file.Raw($"4 0 obj\n<< /Length 20 0 R{(flate ? " /Filter /FlateDecode" : string.Empty)} >>\nstream\n{data}\nendstream\n"); + rows[5] = file.Object(5, "(five)"); + file.Table(21, rows, "/Size 21 /Root 1 0 R"); + var limits = guard == "cut" + ? PdfReaderLimits.Default with { MaxObjectLength = 1024 } + : PdfReaderLimits.Default with { MaxIndexEntries = 20 }; + using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = limits }); + + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + _ = stream.Decode(); + + stream.GetRawBytes().ToArray().Should().Equal(Encoding.Latin1.GetBytes(data)); + stream.Data.CutByGuard.Should().BeFalse(); + document.Reader.IsEndObjMissing(4, out _).Should().BeTrue(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal( + flate + ? [guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries, PdfDiagnosticCodes.FilterFailed] + : [guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries]); + new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.ObjectEndObjMissing).Should().BeTrue(); + } + + [Theory] + [InlineData("cut")] + [InlineData("left out")] + public void An_endstream_only_the_table_follows_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s(string guard) + { + // Object 4, the last, lost its endobj, and the table follows its endstream: data may spell "endstream" and "xref" — + // an embedded file's does —, the stream's own endstream and endobj after, so only the length the guard hid tells + // the two apart. The data up to it is the guard's, the object's endobj unjudged (#224). + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, Page), + [20] = file.Object(20, (guard == "cut" ? new string(' ', 2000) : string.Empty) + "5"), + [4] = file.Position, + }; + file.Raw("4 0 obj\n<< /Length 20 0 R >>\nstream\nBT ET\nendstream\n"); + file.Table(21, rows, "/Size 21 /Root 1 0 R"); + var limits = guard == "cut" + ? PdfReaderLimits.Default with { MaxObjectLength = 1024 } + : PdfReaderLimits.Default with { MaxIndexEntries = 20 }; + using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = limits }); + + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + stream.GetRawBytes().ToArray().Should().Equal("BT ET"u8.ToArray()); + stream.Data.CutByGuard.Should().BeTrue(); + document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); + new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.ObjectEndObjMissing).Should().BeFalse(); + } + [Theory] [InlineData("Rx", false)] [InlineData("RG", false)] From 37ff574f35631a2a29bb6f8350a1505b1f34cac2 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 22:12:53 +0000 Subject: [PATCH 079/146] fix: decode data whose end a guard hides to the bytes its first decode gave A cross-reference stream's data a deferred search took to a spelled endstream, where a guard hides the end, was decoded once more as data a guard cut: ASCII85 and ASCIIHex then drop a last partial group they complete otherwise. Where those bytes alone crossed MaxDecodedStreamLength, the second decode fit, the guard's cut vanished with its report, and the rows past the spelled endstream were charged to the file: a valid file rebuilt, where its direct twin, and the same file under ThrowOnLimit, met the guard. A hidden end now silences only what decoding says of the end, the predictor's short data included; every filter decodes the bytes as the file gives them. Refs #220, #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../IO/Filters/PdfFilterPipeline.cs | 41 +++++++++----- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 54 +++++++++++++++++++ .../DocumentReaderTests.cs | 19 +++++++ 3 files changed, 102 insertions(+), 12 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs index ed8844d4..ac4f9ece 100644 --- a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs +++ b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs @@ -67,7 +67,9 @@ public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diag /// Receives what decoding met; when none are supplied, the stream's document does. /// /// Whether where the data ends is the guard's to hide: data a search took up to an endstream nothing confirmed, - /// the length that would say a guard's. The data is not counted as cut by it. + /// where the length that would say is one a guard cut, or a guard on decoding kept the data from being taken again to + /// it. Only what decoding says of the end hears of it: the data decodes as data whose end the file gives — not as data + /// a guard cut, whose last group a filter drops —, so that whether a guard is reached does not depend on it. /// /// As the other overload gives it. /// As the other overload gives it. @@ -135,9 +137,9 @@ private static ReadOnlyMemory DecodeChain( // Whether the data a filter is given ends where the file's may not: data the reader cut at one of its limits — // MaxStreamLength, or the window a guard stopped at, cut the stream's data, or a filter before it stopped at - // MaxDecodedStreamLength —, or data whose end a guard hides. Every filter after it is given such data, and a cut is - // reported once, by the guard that made it; data whose end is hidden is no cut. - var cut = stream.Data.CutByGuard || endHidden; + // MaxDecodedStreamLength. Every filter after it is given such data, and a cut is reported once, by the guard that + // made it. Data whose end a guard hides is no cut: every filter is given it, and says nothing of its end. + var cut = stream.Data.CutByGuard; var limitReported = false; // The filters before the last share one bound for what they pass along together, and the last keeps its own: a @@ -165,6 +167,7 @@ private static ReadOnlyMemory DecodeChain( diagnostics, stream.Data.Position, guard, + endHidden, ref cut, ref limitReported, out var failed); @@ -203,6 +206,10 @@ public static bool IsImageFilter(PdfName name) => /// Receives what decoding met. /// Where the stream's data starts, where every report is placed. /// Says what reaching the bound does. + /// + /// Whether where ends is the guard's to hide: the filter decodes it as it is, and says nothing of + /// its end. + /// /// /// Whether is data the reader cut; set when this filter stops at the bound too. /// @@ -215,6 +222,7 @@ private static ReadOnlyMemory ApplyOne( PdfDiagnostics? diagnostics, long position, PdfLimitGuard guard, + bool endHidden, ref bool cut, ref bool limitReported, out bool failed) @@ -222,6 +230,10 @@ private static ReadOnlyMemory ApplyOne( var name = step.Name; var maxLength = bounds.MaxLength; var given = cut; + + // Whether how the data ends is the reader's, or the guard's, rather than the file's: data the reader cut, or whose + // end a guard hides. Only what is reported depends on the latter. + var endUnknown = given || endHidden; failed = false; if (name == PdfName.FlateDecode) @@ -243,14 +255,14 @@ private static ReadOnlyMemory ApplyOne( // or a checksum that disagrees, lies in bytes the reader did read, and the whole data would meet it at the // same point: it is the file's. ReportEnding( - given && ending is FlateEnding.TailLost or FlateEnding.ChecksumMissing ? FlateEnding.Whole : ending, + endUnknown && ending is FlateEnding.TailLost or FlateEnding.ChecksumMissing ? FlateEnding.Whole : ending, decoded.Length, faultAt, data.Length, diagnostics, position); Reached(limited, name, bounds.Shared, diagnostics, position, guard, ref cut, ref limitReported); - return ApplyPredictor(decoded, step, diagnostics, position, cut); + return ApplyPredictor(decoded, step, diagnostics, position, cut, endHidden); } if (name == PdfName.LZWDecode) @@ -267,7 +279,7 @@ private static ReadOnlyMemory ApplyOne( $"An LZW stream uses code {undefinedCode}, which it has not defined; what decoded before it was kept."), position); } - else if (ending == DataEnding.Unmarked && !given) + else if (ending == DataEnding.Unmarked && !endUnknown) { // Data the reader cut ends early because the reader stopped reading it: that no code ends it says nothing // of the file. @@ -276,7 +288,7 @@ private static ReadOnlyMemory ApplyOne( "An LZW stream ends without its end-of-data code; it decoded to the end of its data.", position); } - else if (ending == DataEnding.UnmarkedInsideCode && !given) + else if (ending == DataEnding.UnmarkedInsideCode && !endUnknown) { diagnostics?.Warn( PdfDiagnosticCodes.FilterFailed, @@ -285,7 +297,7 @@ private static ReadOnlyMemory ApplyOne( } Reached(limited, name, bounds.Shared, diagnostics, position, guard, ref cut, ref limitReported); - return ApplyPredictor(decoded, step, diagnostics, position, cut); + return ApplyPredictor(decoded, step, diagnostics, position, cut, endHidden); } byte[] text; @@ -308,7 +320,7 @@ private static ReadOnlyMemory ApplyOne( // The end of data the reader cut, or of data the filter stopped decoding at the bound, is the reader's: only what // the filter read is the file's. - ReportFaults(faults, name, data.Length, textLimited, ended: !given && !textLimited, diagnostics, position); + ReportFaults(faults, name, data.Length, textLimited, ended: !endUnknown && !textLimited, diagnostics, position); Reached(textLimited, name, bounds.Shared, diagnostics, position, guard, ref cut, ref limitReported); return text; } @@ -551,8 +563,12 @@ private static void Reached( /// Whether the bound the filters before the last share holds it to less than the bound itself. private readonly record struct Bounds(int MaxLength, bool Shared); + /// + /// Undoes the predictor names, if any: data shorter than one of its rows, whose end the file + /// gives, is left as decoded, and reported unless a guard hides where it ends (). + /// private static ReadOnlyMemory ApplyPredictor( - byte[] decoded, in FilterStep step, PdfDiagnostics? diagnostics, long position, bool cut) + byte[] decoded, in FilterStep step, PdfDiagnostics? diagnostics, long position, bool cut, bool endHidden) { var parameters = step.Parameters; @@ -562,7 +578,8 @@ private static ReadOnlyMemory ApplyPredictor( } if (!PredictorTransform.TryApply( - decoded, parameters.Predictor, parameters.Colors, parameters.BitsPerComponent, parameters.Columns, cut, out var result)) + decoded, parameters.Predictor, parameters.Colors, parameters.BitsPerComponent, parameters.Columns, cut, out var result) && + !endHidden) { diagnostics?.Warn( PdfDiagnosticCodes.FilterFailed, diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 60397c68..c53208be 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -580,6 +580,60 @@ byte[] Data(long four, long stream) => Encoding.ASCII.GetBytes( return file.ToArray(); } + /// + /// A file of two revisions, as writes it, but stream 5's hexadecimal rows hold no + /// stray byte. With , the rows and zero bytes after them are + /// compressed, the zero bytes to a few, and written with no >: /Filter [/ASCIIHexDecode /FlateDecode]. Without, + /// the rows are written as they are, /Filter /ASCIIHexDecode, and an end-of-line, endstream and another + /// follow the digits of 14 bytes and one more: the deferred search stops there. + /// + public static byte[] UpdateOverHexRows(bool deferred, bool compressed, int padding = 0) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + + // Compressed, the data's length depends on the offsets its rows hold: written in four digits, it does not move them. + byte[] Data(long four, long stream) + { + var rows = Encode( + [(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), (1, four, 0), (1, stream, 0)], + Rows.Raw); + + if (compressed) + { + using var output = new MemoryStream(); + + using (var zlib = new ZLibStream(output, CompressionLevel.Optimal, leaveOpen: true)) + { + zlib.Write([.. rows, .. new byte[padding]]); + } + + return Encoding.ASCII.GetBytes(Convert.ToHexString(output.ToArray())); + } + + var hex = Convert.ToHexString(rows); + return Encoding.ASCII.GetBytes(hex[..29] + "\nendstream\n" + hex[29..] + ">"); + } + + var four = file.Position; + var at = four + "4 0 obj\n0000\nendobj\n".Length; + var data = Data(four, at); + var length = data.Length.ToString("D4", CultureInfo.InvariantCulture); + file.Object(4, length); + file.Stream( + 5, + $"/Type /XRef /Size 6 /W [1 4 2] /Filter {(compressed ? "[/ASCIIHexDecode /FlateDecode]" : "/ASCIIHexDecode")} /Root 1 0 R /Length {(deferred ? "4 0 R" : length)}", + data); + file.Text("%" + new string('-', 1200) + "\n"); + var xref = file.Position; + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"xref\n0 4\n0000000000 65535 f\r\n{file.OffsetOf(1):D10} 00000 n\r\n{file.OffsetOf(2):D10} 00000 n\r\n{file.OffsetOf(3):D10} 00000 n\r\ntrailer\n<< /Size 6 /Root 1 0 R /Prev {at} >>\nstartxref\n{xref}\n%%EOF\n")); + return file.ToArray(); + } + /// /// A file of one cross-reference stream, object 7, whose /Length names object 5, which only the stream's own row /// places, before the rows of objects 1 and 2; two rows of reserved types spell endstream after an end-of-line diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 957c16fe..9eaf759a 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -1986,6 +1986,25 @@ static string[] Faults(PdfDocument document) => [.. document.Diagnostics.Where(entry => entry.Code == PdfDiagnosticCodes.FilterFailed).Select(entry => entry.Message)]; } + [Fact] + public void A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s() + { + // Stream 5's hexadecimal rows spell an endstream after the digits of 14 bytes and one more, where the deferred search + // stops: the lone digit makes a fifteenth byte, past MaxDecodedStreamLength. Decoded once more as data a guard cut, + // the digit was dropped and the data fit: the rows past the spelled endstream were charged to the file, the + // section rebuilt, where the direct twin's are the guard's. A hidden end silences what decoding says of the end, + // and changes nothing it decodes (#220, #224). + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxDecodedStreamLength = 14 } }; + using var direct = PdfDocument.Open(ChainFiles.UpdateOverHexRows(deferred: false, compressed: false), options); + using var deferred = PdfDocument.Open(ChainFiles.UpdateOverHexRows(deferred: true, compressed: false), options); + + direct.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitDecodedStream); + deferred.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitDecodedStream); + deferred.WasRepaired.Should().BeFalse(); + deferred.Reader.Structure.Sections[^1].CutByLimit.Should().BeTrue(); + deferred.Reader.Structure.Sections[^1].Incomplete.Should().BeFalse(); + } + [Fact] public void A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is() { From 9c9c1377e04aae3c8ebcf4a1f5725e7046764371 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 22:13:09 +0000 Subject: [PATCH 080/146] fix: hide the end of a deferred length's data only where a guard keeps rows from it The chain decodes once more, its end hidden, data a guard on decoding kept from being taken again to its length. It did so too where the data holds every row the section declares, which nothing would take again: the endstream it was taken to ends it whatever the guard does, and a filter's missing end marker, which the direct twin reports, was lost. The second decode now runs only where rows lack. Refs #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 11 ++++++----- tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 3662c502..76d157bf 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3831,11 +3831,12 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect } } - // Data a /Length the chain could not read left to an endstream nothing confirmed, which a guard on decoding kept - // from being taken again to its length, or taken to what a guard's cut left of the length: where it ends is the - // guard's, which a filter run over data the endstream its rows spell cut short says nothing of the file at. The - // data is decoded once more knowing it, as data a guard cut is: a fault in what a filter read stands (#220, #224). - if (!endHidden && deferred is not null && !xrefStream.Data.CutByGuard && (lengthCut || (!retaken && !decodedWhole))) + // Data a /Length the chain could not read left to an endstream nothing confirmed, where the length that would say + // whether it is the data's own is one a guard cut, or where a guard on decoding kept the data, which lacks rows, + // from being taken again to the length: where it ends is the guard's, and a filter run out there says nothing of + // the file. The data is decoded once more knowing it, to the same bytes, whose faults stand (#220, #224). + if (!endHidden && deferred is not null && !xrefStream.Data.CutByGuard && + (lengthCut || (!retaken && !decodedWhole && declared > held))) { _diagnostics.RollBack(decoding); endHidden = true; diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 9eaf759a..5dac28a1 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -2005,6 +2005,21 @@ public void A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedSt deferred.Reader.Structure.Sections[^1].Incomplete.Should().BeFalse(); } + [Fact] + public void A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends() + { + // Stream 5's rows and 30 zero bytes, compressed, are written in hexadecimal with no ">", and MaxDecodedStreamLength + // stops the Flate output past every row the section declares: no row is missing, so nothing would take the data + // again, and the endstream it was taken to ends it whatever the guard does. Its missing ">" is the file's, as for + // the direct twin (#220). + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxDecodedStreamLength = 50 } }; + using var direct = PdfDocument.Open(ChainFiles.UpdateOverHexRows(deferred: false, compressed: true, padding: 30), options); + using var deferred = PdfDocument.Open(ChainFiles.UpdateOverHexRows(deferred: true, compressed: true, padding: 30), options); + + direct.Diagnostics.Should().Contain(entry => entry.Message.StartsWith("An ASCIIHex stream ends without its > end marker", StringComparison.Ordinal)); + deferred.Diagnostics.Select(entry => entry.Message).Should().Equal(direct.Diagnostics.Select(entry => entry.Message)); + } + [Fact] public void A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is() { From b2cd31c40bf2a6499bf2674af2437126cc40b357 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 22:18:04 +0000 Subject: [PATCH 081/146] fix: reach the guard of an object the chain read through only for a value it keeps The chain loads at opening each object a guard cut after a whole reference that it read through, whichever read asked for it. Two reads keep nothing: an object at a place the section may have been moved to, which the chain refuses as no section, and a cross-reference stream's /Size beside an /Index, which numbers no row. Both now leave the object unloaded, so a valid file opens without the guard's report, and without throwing under ThrowOnLimit. A refused place forgets what it read through, and a kept value that leads to the same object reads it anew. The list is let go too when the chain is rebuilt. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 37 +++++++++++- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 58 +++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 76d157bf..48bb6c21 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -286,8 +286,9 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// /// The objects a guard cut that a cross-reference stream's values led to, which the chain could not read and ended at, and - /// those it read through, the cut having left their value a whole reference: each is loaded once the chain is read, so - /// that its guard is reached where a load reaches it (#263). + /// those it read through, the cut having left their value a whole reference — but for an object it read as a section + /// and refused —: each is loaded once the chain is read, so that its guard is reached where a load reaches it (#263). + /// Let go then, or when the chain is rebuilt. /// private List? _chainValuesCut; @@ -2187,6 +2188,8 @@ private void Initialize() if (!chainRead) { + // A rebuild reads every object again, and loads none the chain read through. + _chainValuesCut = null; Repair(); return; } @@ -2292,6 +2295,26 @@ private void CheckDeferredLengths() } } + /// + /// Forgets the objects read through since held of them, which an + /// object at a place the section may have been moved to, and that is no section, led to: nothing of its reading is kept, + /// and a value the chain keeps that leads to one of them reads it anew, for it to be loaded once the chain is read. + /// + private void ForgetReadThrough(int mark) + { + if (_chainValuesCut is not { } cut) + { + return; + } + + for (var i = mark; i < cut.Count; i++) + { + _readWhileChainIsRead?.Remove(cut[i]); + } + + cut.RemoveRange(mark, cut.Count - mark); + } + /// /// Loads, once the chain is read, each object a guard cut that a cross-reference stream's value led to, which the chain /// ended at or read through: the guard is reached, and reported, where a load reaches it — and thrown, when the caller @@ -3608,6 +3631,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // trailer, and grows its window no further than a trailer may: the data never counts toward it, and a /Length the // chain cannot read has the data searched for its endstream past the window (#174, #220). var guards = _guardsReached; + var readThrough = _chainValuesCut?.Count ?? 0; // A place the section may have been moved to is a guess: what is there and is no section leaves nothing of its // reading, not even a guard its dictionary reached. Where the chain names a section, the guard is reported. @@ -3622,6 +3646,11 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect candidate ? DeclaresCrossReferenceStream : null) || !DeclaresCrossReferenceStream(value)) { + if (candidate) + { + ForgetReadThrough(readThrough); + } + if (!candidate && _guardsReached != guards && value is PdfDictionary { } cut && !cut.ContainsKey(PdfName.Type)) { // A dictionary MaxTrailerLength cut before it said what it is may be the section's: unknown, and the @@ -3753,7 +3782,9 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // one that names an object no section read so far places is UnreadValue's fault. var written = dictionary.GetRaw(PdfName.Index); var index = written?.Resolve() is { } given and not PdfNull ? given : null; - var size = dictionary.GetInteger(PdfName.Size); + + // Beside an /Index, the rows are read without the /Size: it is left as written (UnreadValue). + var size = index is null ? dictionary.GetInteger(PdfName.Size) : null; // The numbering is checked before any row is read: a section that numbers its rows wrongly gives none to believe. // Each value is read once, so that what is read is what was checked. diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 6fef7a4e..9a059ee7 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -975,6 +975,64 @@ public void A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_ document.GetObject(new PdfObjectId(3)).AsDictionary().Should().NotBeNull(); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it(bool kept) + { + // The newer table's /Prev lands 2 bytes into object 8, a stream whose /Length names object 5, which MaxObjectLength + // cuts after "6 0 R", whole: the chain reads object 8 as a section, refuses it, and finds stream 7 after it. What it + // read for object 8 is no section's, and nothing of it was to be kept, but object 5, read through, was loaded at + // opening, the guard reported, or thrown. Where stream 7's /Size names object 5 too, the chain keeps its value, and + // the guard is reached as before (#224, #263). + var file = ChainFiles.UnderAnUpdate( + kept ? "/Type /XRef /Size 5 0 R /W [1 4 2] /Length {length}" : "/Type /XRef /Size 8 /W [1 4 2] /Length {length}", + "6 0 R" + new string(' ', 1100), + compressed: false, + predicted: false, + placed: true, + placedToo: new Dictionary { [6] = "8", [8] = "<< /Length 5 0 R >>\nstream\nabc\nendstream" }); + var text = Encoding.Latin1.GetString(file); + var stream = text.IndexOf("7 0 obj", StringComparison.Ordinal); + var eight = text.IndexOf("8 0 obj", StringComparison.Ordinal); + file = Encoding.Latin1.GetBytes(text.Replace(Invariant($"/Prev {stream} "), Invariant($"/Prev {eight + 2} "), StringComparison.Ordinal)); + + using var document = PdfDocument.Open(file, ObjectGuard); + document.Diagnostics.Select(entry => entry.Code).Should().Equal( + kept ? [PdfDiagnosticCodes.XRefOffsetAdjusted, PdfDiagnosticCodes.LimitObject] : [PdfDiagnosticCodes.XRefOffsetAdjusted]); + + var opening = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }).Dispose(); + + if (kept) + { + opening.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); + } + else + { + opening.Should().NotThrow(); + } + } + + [Fact] + public void A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read() + { + // Stream 7's /Size names object 5, which MaxObjectLength cuts after "9 0 R", whole, beside an /Index the rows are + // numbered by: the chain read it through, though it numbers nothing, and loaded object 5 at opening, reporting the + // guard, or throwing it. It is left as written, as the rows are read without it (#263). + var file = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 5 0 R /Index [0 8] /W [1 4 2] /Length {length}", + "9 0 R" + new string(' ', 1100), + compressed: false, + predicted: false, + placed: true, + placedToo: new Dictionary { [9] = "8" }); + + using var document = PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); + + document.Diagnostics.Should().BeEmpty(); + document.Reader.Structure.Sections.Should().HaveCount(2).And.OnlyContain(section => section.State == IO.XRef.XRefSectionState.Read); + } + [Fact] public void Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value() { From 78386ba49b8741a87f5922d4dffa26447976e855 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 22:23:50 +0000 Subject: [PATCH 082/146] fix: judge a cross-reference stream's parameters as decoding read them, whatever the cache let go The rule on trailer values follows a parameter only where decoding does, keyed on the references decoding read. It resolved /DecodeParms a second time to walk it: where the cache had let the dictionary go, it was parsed again, its references other instances, and a parameter decoding read was judged as written, the real it leads to unseen. The rule now walks what decoding's reads led to. A test holds it to the bound on filters the reader read under, which no test did. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../Validation/Rules/TrailerValueWrongRule.cs | 21 ++++--- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 55 +++++++++++++++++++ 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs b/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs index f2a4babe..54f292f8 100644 --- a/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs +++ b/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs @@ -37,7 +37,8 @@ namespace AdCodicem.Pdf.Validation.Rules; /// which the reader does not follow, the chain going on through the table's; the /Prev and /XRefStm of a /// table an /XRefStm names in a stream's place; and what of a cross-reference stream's /Filter and /// /DecodeParms decoding does not read — the parameters of a filter that reads none, past its predictor where it -/// has none, or past the element that ends the chain. +/// has none, or past the element that ends the chain —, what decoding reads judged as it read it, whatever the cache let +/// go since. /// /// internal sealed class TrailerValueWrongRule : IValidationRule @@ -248,27 +249,31 @@ private void Parameters(PdfDictionary parameters, string where, string? inside, /// /// The references among a cross-reference stream's /Filter and /DecodeParms that decoding follows, each as - /// the file wrote it, which gives in the order decoding reads them (#263). + /// the file wrote it, which gives in the order decoding reads them, with what it led + /// to (#263). The rule walks what they led to, not what a second read of them gives: the cache may have let an object go + /// since, and its references read again are other instances, which decoding did not follow. /// private readonly struct DecodingReads() : IFilterValueReader { - private readonly HashSet _followed = new(ReferenceEqualityComparer.Instance); + private readonly Dictionary _followed = new(ReferenceEqualityComparer.Instance); /// Determines whether is no reference, or one decoding follows. - public bool Follows(PdfObject? written) => written is not PdfReference || _followed.Contains(written); + public bool Follows(PdfObject? written) => written is not PdfReference || _followed.ContainsKey(written); - /// Gives what leads to where decoding follows it, and itself otherwise. - public PdfObject? Resolve(PdfObject? written) => Follows(written) ? written?.Resolve() : written; + /// Gives what led to where decoding follows it, and itself otherwise. + public PdfObject? Resolve(PdfObject? written) => + written is PdfReference && _followed.TryGetValue(written, out var value) ? value : written; /// public bool TryRead(PdfName key, PdfObject? written, bool entry, out PdfObject? value) { + value = written?.Resolve(); + if (written is PdfReference) { - _followed.Add(written); + _followed[written] = value; } - value = written?.Resolve(); return true; } } diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 9a059ee7..dfb6b598 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1486,6 +1486,61 @@ public void The_rule_on_trailer_values_follows_a_parameter_only_where_decoding_r guarded.Diagnostics.Should().BeEmpty(); } + [Fact] + public void The_rule_on_trailer_values_follows_no_parameter_past_the_step_MaxFilterChainLength_ends_decoding_at() + { + // Stream 7's second Flate step reads /Columns past its predictor, in object 6, which holds 7.0; MaxFilterChainLength + // lets the first step alone run. Decoding never read the parameter, and the rule, held to the bound the reader read + // under, judges nothing it leads to (#263). + var file = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] /Filter [/FlateDecode /FlateDecode] /DecodeParms [null 5 0 R] /Length {length}", + "<< /Predictor 12 /Columns 6 0 R >>", + compressed: true, + predicted: false, + placed: true, + placedToo: new Dictionary { [6] = "7.0" }); + using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxFilterChainLength = 1 } }); + var trailers = new ValidationProfile("trailers", 1, [new TrailerValueWrongRule()]); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitFilterChain); + new PdfValidator(new PdfValidatorOptions { Profile = trailers }).Validate(document).Findings.Should().BeEmpty(); + } + + [Fact] + public void The_rule_on_trailer_values_judges_a_parameter_decoding_read_whatever_the_cache_let_go_before_validating() + { + // Stream 7's /DecodeParms, object 5, gives /Columns past its predictor as a reference to object 100, which holds 7.0. + // Read before validating, with 63 objects after it, object 5 left a cache of 64: the rule read it again, a reference + // to /Columns decoding had not followed, and lost the finding decoding's own read gives (#263). + var placed = new Dictionary { [100] = "7.0" }; + + for (var number = 200; number < 270; number++) + { + placed[number] = "(x)"; + } + + var file = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] /Filter /FlateDecode /DecodeParms 5 0 R /Length {length}", + "<< /Predictor 12 /Columns 100 0 R >>", + compressed: true, + predicted: true, + placed: true, + placedToo: placed); + using var document = PdfDocument.Open(file, new PdfReaderOptions { ObjectCacheCapacity = 64 }); + _ = document.GetObject(new PdfObjectId(5)); + + for (var number = 200; number < 263; number++) + { + _ = document.GetObject(new PdfObjectId(number)); + } + + var trailers = new ValidationProfile("trailers", 1, [new TrailerValueWrongRule()]); + + new PdfValidator(new PdfValidatorOptions { Profile = trailers }).Validate(document).Findings.Select(finding => finding.Message) + .Should().ContainSingle(message => message.EndsWith( + "gives /Columns of its /DecodeParms as a real number, where Table 8 of ISO 32000-1 asks for an integer.", StringComparison.Ordinal)); + } + [Theory] [InlineData("Prev", "9 0 R", true)] [InlineData("XRefStm", "9 0 R", true)] From a06681e0ac45957d4a6bd2061407544686cd1728 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 22:23:51 +0000 Subject: [PATCH 083/146] fix: judge a cross-reference stream's /W element past the third as written The reader reads three elements of a /W, the widths of three fields. The rule on trailer values followed every element: one past the third, written as a reference, was loaded at validation, which could reach a guard the reading never met, and said to lead to a real the reader never read. It is now judged as written, as other values the reader does not read are. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../Validation/Rules/TrailerValueWrongRule.cs | 12 +++++----- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 22 +++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs b/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs index 54f292f8..7d00cdd4 100644 --- a/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs +++ b/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs @@ -35,10 +35,10 @@ namespace AdCodicem.Pdf.Validation.Rules; /// reader does not read is judged as the file wrote it, a reference it holds not followed: following it would read what /// the reading never did, and could reach a guard it never met (#263). So are a hybrid file's stream's own /Prev, /// which the reader does not follow, the chain going on through the table's; the /Prev and /XRefStm of a -/// table an /XRefStm names in a stream's place; and what of a cross-reference stream's /Filter and -/// /DecodeParms decoding does not read — the parameters of a filter that reads none, past its predictor where it -/// has none, or past the element that ends the chain —, what decoding reads judged as it read it, whatever the cache let -/// go since. +/// table an /XRefStm names in a stream's place; the elements of a /W past the third; and what of a +/// cross-reference stream's /Filter and /DecodeParms decoding does not read — the parameters of a filter +/// that reads none, past its predictor where it has none, or past the element that ends the chain —, what decoding reads +/// judged as it read it, whatever the cache let go since. /// /// internal sealed class TrailerValueWrongRule : IValidationRule @@ -193,9 +193,11 @@ public void Integers(PdfDictionary trailer, PdfName key) return; } + // The reader reads three elements of a /W, the widths of three fields, and every element of an /Index. for (var i = 0; i < array.Count; i++) { - Value(string.Create(CultureInfo.InvariantCulture, $"element {i} of its {name}"), array[i], Iso1, "Table 17"); + Value( + string.Create(CultureInfo.InvariantCulture, $"element {i} of its {name}"), array[i], Iso1, "Table 17", key != PdfName.W || i < 3); } } diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index dfb6b598..ecad1168 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1541,6 +1541,28 @@ public void The_rule_on_trailer_values_judges_a_parameter_decoding_read_whatever "gives /Columns of its /DecodeParms as a real number, where Table 8 of ISO 32000-1 asks for an integer.", StringComparison.Ordinal)); } + [Theory] + [InlineData("7.0")] + [InlineData("padded")] + public void The_rule_on_trailer_values_judges_a_w_element_past_the_third_as_written(string five) + { + // Stream 7's /W gives a fourth element, a reference to object 5, which holds 7.0, or 7 past 3,000 spaces, which + // MaxObjectLength cuts. The reader reads three: the rule followed the fourth all the same, judged a real the reading + // never read, and reached the guard the reading never met. It is judged as written, a reference (#263). + var file = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2 5 0 R] /Length {length}", + five == "7.0" ? five : new string(' ', 3000) + "7", + compressed: false, + predicted: false, + placed: true); + using var document = PdfDocument.Open(file, ObjectGuard); + var trailers = new ValidationProfile("trailers", 1, [new TrailerValueWrongRule()]); + + new PdfValidator(new PdfValidatorOptions { Profile = trailers }).Validate(document).Findings.Select(finding => finding.Message) + .Should().ContainSingle().Which.Should().EndWith("writes element 3 of its /W as the reference 5 0 R, where ISO 32000-1 makes it direct (7.5.8.2)."); + document.Diagnostics.Should().BeEmpty(); + } + [Theory] [InlineData("Prev", "9 0 R", true)] [InlineData("XRefStm", "9 0 R", true)] From f3810a3dd9912548239045379e1a6313fc05958a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 22:31:39 +0000 Subject: [PATCH 084/146] fix: find a cross-reference stream a guard cut where it was moved; keep nothing of a place dropped Near an offset a section is misnamed at, the chain kept only a place it could read. A cross-reference stream whose /W, /Index or another value leads to an object MaxObjectLength cuts is the section all the same, kept from being read by the guard: it was dropped, said to be missing, and the index rebuilt for the guard's cut. It is now found there, the offset's fault repaired, and the chain ends at it as the guard's, as where it is named exactly. A place dropped, for whatever reason, forgets what its values read through, which was loaded once the chain was read. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 32 ++++++---- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 58 +++++++++++++++++++ 2 files changed, 78 insertions(+), 12 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 48bb6c21..ea23534c 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -2296,9 +2296,9 @@ private void CheckDeferredLengths() } /// - /// Forgets the objects read through since held of them, which an - /// object at a place the section may have been moved to, and that is no section, led to: nothing of its reading is kept, - /// and a value the chain keeps that leads to one of them reads it anew, for it to be loaded once the chain is read. + /// Forgets the objects read through since held of them, which a + /// place the section may have been moved to, and that is dropped, led to: nothing of its reading is kept, and a value + /// the chain keeps that leads to one of them reads it anew, for it to be loaded once the chain is read. /// private void ForgetReadThrough(int mark) { @@ -2924,7 +2924,8 @@ private void ReportMissingSection(string naming, long offset, long namedFrom) /// places in all, over the whole document: each is read through a window /// of up to 64 KB, the file chooses how many headers lie near a section, and a chain can name a section for /// each of its links. Found, the section's record takes what - /// was read there; not found, it keeps what the named offset held. + /// was read there — a section a guard kept from being read too, which is not found to be read, the chain ending + /// there as the guard's —; not found, it keeps what the named offset held. /// private bool TryRelocateXRefSection(long offset, XRefSectionRecord section, out long previous, out long hybrid) { @@ -2962,8 +2963,12 @@ private bool TryRelocateXRefSection(long offset, XRefSectionRecord section, out } var attempt = new XRefSectionRecord(section.NamedBy, candidate, section.NamedFrom); + var readThrough = _chainValuesCut?.Count ?? 0; + var state = TryReadXRefSection(candidate, attempt, out previous, out hybrid, candidate: true); - if (TryReadXRefSection(candidate, attempt, out previous, out hybrid, candidate: true) == XRefSectionState.Read) + // A section a guard kept from being read is the section all the same: found where it was moved, it is where a + // guard cut the chain, as where it is named (#263). Any other place is dropped with what it read through. + if (state == XRefSectionState.Read || (state == XRefSectionState.Malformed && attempt.CutByLimit)) { section.RelocateTo(attempt); _diagnostics.Repair( @@ -2972,8 +2977,17 @@ private bool TryRelocateXRefSection(long offset, XRefSectionRecord section, out CultureInfo.InvariantCulture, $"The cross-reference section {section.NamedBy} names at offset {absolute} was found {candidate - absolute} bytes from there."), candidate); - return true; + + if (state == XRefSectionState.Read) + { + return true; + } + + section.State = XRefSectionState.Malformed; + break; } + + ForgetReadThrough(readThrough); } previous = -1; @@ -3631,7 +3645,6 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // trailer, and grows its window no further than a trailer may: the data never counts toward it, and a /Length the // chain cannot read has the data searched for its endstream past the window (#174, #220). var guards = _guardsReached; - var readThrough = _chainValuesCut?.Count ?? 0; // A place the section may have been moved to is a guess: what is there and is no section leaves nothing of its // reading, not even a guard its dictionary reached. Where the chain names a section, the guard is reported. @@ -3646,11 +3659,6 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect candidate ? DeclaresCrossReferenceStream : null) || !DeclaresCrossReferenceStream(value)) { - if (candidate) - { - ForgetReadThrough(readThrough); - } - if (!candidate && _guardsReached != guards && value is PdfDictionary { } cut && !cut.ContainsKey(PdfName.Type)) { // A dictionary MaxTrailerLength cut before it said what it is may be the section's: unknown, and the diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index ecad1168..7f865086 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1625,6 +1625,64 @@ public void A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is key == "Prev" ? "names older sections" : "names the cross-reference stream of its revision", StringComparison.Ordinal)); } + [Theory] + [InlineData("/W 5 0 R", "[1 4 2]")] + [InlineData("/W [1 4 2] /Index 5 0 R", "[0 8]")] + public void A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved(string entries, string value) + { + // The newer table's /Prev lands 3 bytes into stream 7, whose /W or /Index names object 5, which MaxObjectLength cuts + // before its value. The stream, found 3 bytes away, was dropped as no section, since the guard kept it from being + // read: the section was said to be missing, and the index rebuilt, for the guard's cut. It is the section moved, + // which the guard ends the chain at, as where the /Prev names it exactly (#263). + var file = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 " + entries + " /Length {length}", new string(' ', 1524) + value, compressed: false, predicted: false, placed: true); + var text = Encoding.Latin1.GetString(file); + var stream = text.IndexOf("7 0 obj", StringComparison.Ordinal); + file = Encoding.Latin1.GetBytes(text.Replace(Invariant($"/Prev {stream} "), Invariant($"/Prev {stream + 3} "), StringComparison.Ordinal)); + + using (var whole = PdfDocument.Open(file)) + { + whole.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.XRefOffsetAdjusted); + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.XRefOffsetAdjusted, PdfDiagnosticCodes.LimitObject); + document.WasRepaired.Should().BeFalse(); + var moved = document.Reader.Structure.Sections[^1]; + moved.Offset.Should().Be(stream); + moved.CutByLimit.Should().BeTrue(); + var findings = new PdfValidator().Validate(document).Findings.Select(finding => finding.RuleId); + findings.Should().Contain(PdfValidationRuleIds.XRefCheckedInPart).And.NotContain(PdfValidationRuleIds.XRefSectionNotFound); + } + + [Fact] + public void A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild() + { + // The newer table's /Prev lands 3 bytes into stream 7, whose /W names object 5, which MaxObjectLength cuts after + // "6 0 R", whole, and whose /Index names object 9, which nothing places: the stream, read through object 5, cannot be + // read, the file's fault, and is no section. Object 5 was loaded once the chain was read all the same: nothing of a + // place dropped is kept. The index is rebuilt, as read whole, and the rebuild, which reads object 5, reaches the + // guard (#263). + var file = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W 5 0 R /Index 9 0 R /Length {length}", + "6 0 R" + new string(' ', 1524), + compressed: false, + predicted: false, + placed: true, + placedToo: new Dictionary { [6] = "[1 4 2]" }); + var text = Encoding.Latin1.GetString(file); + var stream = text.IndexOf("7 0 obj", StringComparison.Ordinal); + file = Encoding.Latin1.GetBytes(text.Replace(Invariant($"/Prev {stream} "), Invariant($"/Prev {stream + 3} "), StringComparison.Ordinal)); + + using var whole = PdfDocument.Open(file); + using var document = PdfDocument.Open(file, ObjectGuard); + + whole.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.XRefSectionMissing, PdfDiagnosticCodes.XRefRebuilt); + document.Diagnostics.Select(entry => entry.Code).Should().Equal( + PdfDiagnosticCodes.XRefSectionMissing, PdfDiagnosticCodes.XRefRebuilt, PdfDiagnosticCodes.LimitObject); + } + [Fact] public void A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain() { From 0bfee24a4735e1d208ee587ca12b0ff367919dd5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 06:59:54 +0000 Subject: [PATCH 085/146] fix: under a hidden length, end a stream at an endstream only where what follows proves it its own Where a guard cut a stream's /Length or left it out of the index, the search for its endstream stops at one the data may spell. An endstream its object's endobj follows was taken for the stream's own: data spelling "endstream" and "endobj" in a string was cut short there, unmarked, its filter said to end early. Only what follows the object's end can tell: the next object the index places, the end of the file, or the xref or startxref keyword, after the endobj or after the endstream where none follows. Anything else leaves the data to the guard, the object's endobj unjudged. The proof is read once for each place, within what the searches may read, and replaces the narrower rule earlier in this batch, which an endstream the table follows did not meet. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../IO/IPdfStreamDataProvider.cs | 13 ++ src/AdCodicem.Pdf/IO/PdfFileReader.cs | 63 ++++++--- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 14 +- src/AdCodicem.Pdf/IO/StreamEndSearch.cs | 7 - .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 132 ++++++++++++++++-- 5 files changed, 188 insertions(+), 41 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs index 92e1ab3b..6904a015 100644 --- a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs +++ b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs @@ -72,6 +72,19 @@ internal interface IPdfStreamDataProvider : IPdfFileEdge /// StreamEndSearch FindEndStream(int number, long absoluteDataStart, long declaredEnd, ReadOnlySpan buffered, bool deferred); + /// + /// Determines whether the endstream that ends data at , which a search found, + /// is the stream's own beyond what its length could say: what follows its object's end — its endobj, where + /// says one follows it, or the endstream itself otherwise —, white space and comments + /// between, is the next object the index places, , the end of the file, or the + /// xref or startxref keyword. Anything else may follow an endstream the data spells, which only + /// the length tells from the stream's own (#224). + /// + /// + /// Read once for each place, as far as the searches for endstream may read; past that, nothing is proven. + /// + bool IsOwnEndStream(long absoluteDataEnd, EndObjState endObj, long? nextObject); + /// /// Resolves the object a stream's /Length names, and says whether the file holds it and whether it could be /// read — a reference to the stream itself, or to an object inside an object stream being decoded, cannot be —, or diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index ea23534c..fbabc49a 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -388,6 +388,12 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// private readonly Dictionary _endStreamChecks = []; + /// + /// Whether the endstream after each place a search ended data at is the stream's own beyond what its length + /// could say (): read once, however many times its stream is parsed. + /// + private readonly Dictionary _ownEndStreams = []; + /// /// Where the data starts of each stream whose length fault a reading the reader kept reported: the stream is not /// reported again when it is parsed again. @@ -1872,31 +1878,54 @@ private StreamEndSearch SearchEndStreamOnce(int number, long absoluteDataStart, // Past int.MaxValue bytes, no length the reader holds a stream by reaches the endstream found: only a file of more // than 2 GB can put one there, and the stream is read as one whose endstream is not found. With none, what the // object's endobj does is read once with the search, as long as the searches may read. - if (dataEnd >= 0 && dataEnd - absoluteDataStart <= int.MaxValue) - { - var endObj = ((IPdfStreamDataProvider)this).CheckEndStream(dataEnd) ?? EndObjState.Unseen; - return new StreamEndSearch((int)(dataEnd - absoluteDataStart), stop, endObj) - { - NextObjectFollows = endObj == EndObjState.Absent && nextObject is { } next && EndStreamLeadsTo(dataEnd, next), + return dataEnd >= 0 && dataEnd - absoluteDataStart <= int.MaxValue + ? new StreamEndSearch( + (int)(dataEnd - absoluteDataStart), + stop, + ((IPdfStreamDataProvider)this).CheckEndStream(dataEnd) ?? EndObjState.Unseen) + : new StreamEndSearch(null, stop, EndObjState.Unknown, searched) + { + EndObjSeen = endObjSeen, + ObjectEnd = objectEnd, + HeaderStopped = header >= 0, }; + } + + /// + bool IPdfStreamDataProvider.IsOwnEndStream(long absoluteDataEnd, EndObjState endObj, long? nextObject) + { + if (_ownEndStreams.TryGetValue(absoluteDataEnd, out var known)) + { + return known; } - return new StreamEndSearch(null, stop, EndObjState.Unknown, searched) + // The endstream keyword, then the endobj that follows it, where one does: each found where a reading said it lies. + var end = NextSignificant(absoluteDataEnd); + end = end < 0 || end >= _source.Length ? -1 : end + StreamEnds.EndStreamKeyword.Length; + + if (end >= 0 && endObj == EndObjState.Present) { - EndObjSeen = endObjSeen, - ObjectEnd = objectEnd, - HeaderStopped = header >= 0, - }; + end = NextSignificant(end); + end = end < 0 || end >= _source.Length ? -1 : end + StreamEnds.EndObjKeyword.Length; + } + + var next = end < 0 ? -1 : NextSignificant(end); + var own = next >= 0 && + (next == _source.Length || next == nextObject || IsKeywordAt(next, "xref"u8) || IsKeywordAt(next, StartXRefKeyword)); + _ownEndStreams[absoluteDataEnd] = own; + return own; } /// - /// Determines whether the first token after the endstream that ends data at , white space - /// and comments between, starts at , as far as the searches may read. + /// Determines whether , whole, starts at : the file ends after it, + /// or no regular character follows. /// - private bool EndStreamLeadsTo(long dataEnd, long next) + private bool IsKeywordAt(long position, ReadOnlySpan keyword) { - var keyword = NextSignificant(dataEnd); - return keyword >= 0 && NextSignificant(keyword + StreamEnds.EndStreamKeyword.Length) == next; + Span bytes = stackalloc byte[StartXRefKeyword.Length + 1]; + var length = Math.Min(keyword.Length + 1, _source.Length - position); + var read = _source.Read(position, bytes[..(int)length]); + return bytes[..read].StartsWith(keyword) && (read == keyword.Length ? position + read >= _source.Length : !PdfCharacters.IsRegular(bytes[keyword.Length])); } /// @@ -2153,6 +2182,8 @@ internal void Close() _endStreamSearches.TrimExcess(); _endStreamChecks.Clear(); _endStreamChecks.TrimExcess(); + _ownEndStreams.Clear(); + _ownEndStreams.TrimExcess(); Release(_lengthFaultsReported); Release(_syntaxReported); Release(_limitsReached); diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index fe3d0e93..337cd511 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -833,12 +833,14 @@ private PdfStream ReadStream(PdfDictionary dictionary) WarnLengthFault(PdfDiagnosticCodes.StreamLengthInvalid, DescribeRecovered(declared, found), dataStart); } - // An endstream its object's endobj does not follow, where a guard cut the /Length or left it out of the index: one - // the data spells, as a valid stream's may, ends the search as the stream's own would, its object's endobj lost — - // the length that tells them apart is the guard's. The data up to it is the guard's, cut by it, the object's - // endobj unjudged, and nothing is said of the file, as at a header the data spells (#224). One the next object - // the index places follows is the stream's own, which no data spells: its object lacks the endobj. - if (hidden && settled.EndObj == EndObjState.Absent && !settled.NextObjectFollows) + // Where a guard cut the /Length or left it out of the index, an endstream the data spells, as a valid stream's may — + // in a string, followed by endobj or not —, ends the search as the stream's own would: the length that tells them + // apart is the guard's. One what follows proves its own — the next object the index places, the end of the file, + // a cross-reference section's keyword, after the object's endobj or in its place — ends the data, the file's. + // Any other's data is the guard's, cut by it, the object's endobj unjudged, and nothing is said of the file, as + // at a header the data spells (#224). + if (hidden && settled.EndObj is EndObjState.Absent or EndObjState.Present && _streamData is { } file && + !file.IsOwnEndStream(_baseOffset + dataStart + found, settled.EndObj, settled.NextObject)) { _endObjUnknown = true; return Finish(dictionary, dataStart, found, span.Length, cutByGuard: true); diff --git a/src/AdCodicem.Pdf/IO/StreamEndSearch.cs b/src/AdCodicem.Pdf/IO/StreamEndSearch.cs index 42984ace..0d5debdd 100644 --- a/src/AdCodicem.Pdf/IO/StreamEndSearch.cs +++ b/src/AdCodicem.Pdf/IO/StreamEndSearch.cs @@ -45,11 +45,4 @@ internal readonly record struct StreamEndSearch(int? Length, long? NextObject, E /// then gives, rather than the next object the index places or the end of the file. /// public bool HeaderStopped { get; init; } - - /// - /// Gets a value indicating whether the next object the index places follows the endstream found, white space and - /// comments between: no endstream the data spells can be followed by an object the index places, so the one found - /// is the stream's own, and the endobj it lacks the file's, whatever hid the length (#224). - /// - public bool NextObjectFollows { get; init; } } diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 7f865086..495b6855 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -470,6 +470,64 @@ public void A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_gua throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); } + /// + /// A page whose content stream, object 4, holds BT ET, its /Length object 20, which holds 5 after 2,000 + /// spaces where is cut; its endstream, and its endobj where + /// , followed by : object 6, which the table places, or which no row + /// places; the table; the startxref after the table, which comes before object 4; or the end of the file, past + /// the one the table's %%EOF marks. + /// + private static byte[] StreamFollowedBy(string guard, string follower, bool endObj) + { + var four = "4 0 obj\n<< /Length 20 0 R >>\nstream\nBT ET\nendstream\n" + (endObj ? "endobj\n" : string.Empty); + var late = follower is "startxref" or "end of file"; + + if (!late) + { + return Encoding.Latin1.GetBytes(Text(0)); + } + + // The table's row for object 4 gives where it lies after the table: an offset as many digits long wherever it lies. + var first = Text(0); + var at = follower == "startxref" ? first.LastIndexOf("startxref", StringComparison.Ordinal) : first.Length; + return Encoding.Latin1.GetBytes(Text(at).Insert(at, four)); + + string Text(long at) + { + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, Page), + [20] = file.Object(20, (guard == "cut" ? new string(' ', 2000) : string.Empty) + "5"), + }; + + if (late) + { + rows[4] = at; + } + else + { + rows[4] = file.Position; + file.Raw(four); + + if (follower != "xref") + { + var six = file.Object(6, "(six)"); + + if (follower == "next object") + { + rows[6] = six; + } + } + } + + file.Table(21, rows, "/Size 21 /Root 1 0 R"); + return Encoding.Latin1.GetString(file.Bytes); + } + } + /// /// A page whose content stream, object 4, holds , its /Length object 20, which holds /// ; with no endstream unless . @@ -779,35 +837,85 @@ public void An_endstream_the_next_object_follows_where_a_guard_hid_the_length_la } [Theory] - [InlineData("cut")] - [InlineData("left out")] - public void An_endstream_only_the_table_follows_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s(string guard) + [InlineData("next object", true)] + [InlineData("next object", false)] + [InlineData("xref", true)] + [InlineData("xref", false)] + [InlineData("startxref", true)] + [InlineData("startxref", false)] + [InlineData("end of file", true)] + [InlineData("end of file", false)] + [InlineData("object no row places", true)] + [InlineData("object no row places", false)] + public void An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own( + string follower, bool endObj) + { + // Object 4's /Length, object 20, is one MaxObjectLength cuts or MaxIndexEntries leaves out of the index, and its + // endstream, which its endobj follows or not, is followed by the next object the index places, a cross-reference + // section's keyword, or the end of the file: no endstream the data spells is, and the data is the file's, the + // object's endobj judged. Followed by an object no row places, the endstream may be one the data spells, which only + // the length tells from the stream's own: the data up to it is the guard's, the object's endobj unjudged (#224). + var proven = follower != "object no row places"; + + foreach (var guard in (string[])["cut", "left out"]) + { + var limits = guard == "cut" + ? PdfReaderLimits.Default with { MaxObjectLength = 1024 } + : PdfReaderLimits.Default with { MaxIndexEntries = 20 }; + using var document = PdfDocument.Open(StreamFollowedBy(guard, follower, endObj), new PdfReaderOptions { Limits = limits }); + + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + stream.GetRawBytes().ToArray().Should().Equal("BT ET"u8.ToArray()); + stream.Data.CutByGuard.Should().Be(!proven, guard); + document.Reader.IsEndObjMissing(4, out _).Should().Be(proven && !endObj, guard); + document.Diagnostics.Select(entry => entry.Code).Should().Equal( + guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries); + } + } + + [Theory] + [InlineData("cut", false)] + [InlineData("left out", false)] + [InlineData("cut", true)] + [InlineData("left out", true)] + public void An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s(string guard, bool hex) { - // Object 4, the last, lost its endobj, and the table follows its endstream: data may spell "endstream" and "xref" — - // an embedded file's does —, the stream's own endstream and endobj after, so only the length the guard hid tells - // the two apart. The data up to it is the guard's, the object's endobj unjudged (#224). + // Object 4's content spells "endstream" and "endobj" in a string, which more of the content follows: read whole, the + // length passes them. Where a guard hid it, the search stopped there, the endobj taken for the object's own and the + // data, cut short, for the file's — its ASCIIHex data said to lack its ">". Nothing after them proves the endstream + // the stream's own: the data up to it is the guard's (#224). + const string Spelled = "\nendstream\nendobj\n"; + var data = hex ? "4254202861" + Spelled + "29205466>" : "BT (a" + Spelled + ") Tf ET"; var file = new HandWrittenFile(); var rows = new Dictionary { [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), [3] = file.Object(3, Page), - [20] = file.Object(20, (guard == "cut" ? new string(' ', 2000) : string.Empty) + "5"), - [4] = file.Position, + [4] = file.Object(4, $"<< /Length 20 0 R{(hex ? " /Filter /ASCIIHexDecode" : string.Empty)} >>\nstream\n{data}\nendstream"), + [6] = file.Object(6, "(six)"), + [20] = file.Object(20, (guard == "cut" ? new string(' ', 2000) : string.Empty) + data.Length.ToString(CultureInfo.InvariantCulture)), }; - file.Raw("4 0 obj\n<< /Length 20 0 R >>\nstream\nBT ET\nendstream\n"); file.Table(21, rows, "/Size 21 /Root 1 0 R"); var limits = guard == "cut" ? PdfReaderLimits.Default with { MaxObjectLength = 1024 } : PdfReaderLimits.Default with { MaxIndexEntries = 20 }; - using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = limits }); + using (var whole = PdfDocument.Open(file.Bytes)) + { + whole.GetObject(new PdfObjectId(4)).AsStream().Required().GetRawBytes().ToArray().Should().Equal(Encoding.Latin1.GetBytes(data)); + } + + using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = limits }); var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + _ = stream.Decode(); - stream.GetRawBytes().ToArray().Should().Equal("BT ET"u8.ToArray()); + stream.GetRawBytes().ToArray().Should().Equal(Encoding.Latin1.GetBytes(data[..data.IndexOf('\n', StringComparison.Ordinal)])); stream.Data.CutByGuard.Should().BeTrue(); document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); - new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.ObjectEndObjMissing).Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal( + guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries); } [Theory] From 51eac1d08532c7ddf6e2a0c6983c147f40146e1d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 07:16:18 +0000 Subject: [PATCH 086/146] fix: leave the rows a cross-reference stream lacks to the file where its endstream is proven its own Where a guard cut a cross-reference stream's /Length, which its own row places, the chain took its data to the first endstream and the rows it lacks for the guard's, its end hidden: a filter run out there said nothing. A load of the same stream, its endstream proven its own by what follows, reads the file's end and reports the run-out. The chain now asks the same proof of the endstream the data was taken, or taken again, to: proven, the rows the data lacks are the file's, the section incomplete and the index rebuilt, as read whole; otherwise they stay the guard's. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 24 ++++++++++++++++- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 13 ++++++--- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 27 +++++++++++++++++++ 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index fbabc49a..01ed2c65 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3890,7 +3890,13 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // is asked too: a newer section may place the object, which a guard cut. if (!retaken && !endHidden && deferred is { } unsettled && declared > held && decodedWhole) { - if (RetakeDeferred(xrefStream, unsettled, data, fieldWidths, row, held, index is null ? [0, declared] : subsections, out lengthCut) is { } whole) + var whole = RetakeDeferred(xrefStream, unsettled, data, fieldWidths, row, held, index is null ? [0, declared] : subsections, out lengthCut); + + // Where a guard cut the length, the endstream the data ends at is the stream's own, and the rows it lacks + // the file's, only where what follows proves it, as a load of the stream reads it (#224). + lengthCut = lengthCut && !EndsAtOwnEndStream(whole ?? xrefStream); + + if (whole is not null) { _diagnostics.RollBack(decoding); xrefStream = whole; @@ -3994,6 +4000,22 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect return XRefSectionState.Read; } + /// + /// Determines whether the endstream cross-reference stream 's data ends at, as the chain + /// took it or took it again, is the stream's own beyond what a /Length a guard cut could say + /// (): where the data ends is then the file's (#224). + /// + private bool EndsAtOwnEndStream(PdfStream stream) + { + var dataStart = stream.Data.Position; + var dataEnd = dataStart + stream.Data.Length; + var provider = (IPdfStreamDataProvider)this; + + return _endStreamSearches.TryGetValue((dataStart, true), out var known) && + provider.CheckEndStream(dataEnd) is EndObjState endObj and (EndObjState.Absent or EndObjState.Present) && + provider.IsOwnEndStream(dataEnd, endObj, known.Search.NextObject); + } + /// /// Gives cross-reference stream with its data taken again to the length its /Length /// gives, which the chain could not read and took the data up to the first endstream for, when the rows diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index c53208be..9e749a42 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -123,8 +123,10 @@ public static byte[] UnderAnUpdate( /// free rows; with , a row that gives the next /// object free with generation 13, so that it ends in a carriage return; then, with , /// two rows of reserved types that spell endstream after an end-of-line, or with - /// an object header. The rows are compressed when , and the - /// data ends with before its endstream. + /// an object header. The rows are compressed when , the + /// data loses its last bytes, and it ends with before its + /// endstream, which its endobj and the startxref follow. The object that holds the length holds + /// spaces before it. /// public static byte[] AloneWithIndirectLength( int lengthOff = 0, @@ -134,7 +136,9 @@ public static byte[] AloneWithIndirectLength( string endOfLine = "\n", int padding = 0, bool lengthLast = false, - bool spellingHeader = false) + bool spellingHeader = false, + int dropped = 0, + int lengthPadding = 0) { using var file = new Writer("1.5"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); @@ -147,7 +151,7 @@ public static byte[] AloneWithIndirectLength( // Uncompressed rows, or stored in blocks of their own, take as many bytes whatever they hold. var length = Data(0, 0).Length + lengthOff; - file.Object(lengthNumber, length.ToString(CultureInfo.InvariantCulture)); + file.Object(lengthNumber, new string(' ', lengthPadding) + length.ToString(CultureInfo.InvariantCulture)); var stream = file.Position; var filter = compressed ? " /Filter /FlateDecode" : string.Empty; file.Stream( @@ -183,6 +187,7 @@ byte[] Data(long lengthAt, long streamAt) data.AddRange(Encode([(1, lengthAt, 0)], Rows.Raw)); } + data.RemoveRange(data.Count - dropped, dropped); return [.. data]; } } diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 495b6855..278337bb 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -874,6 +874,33 @@ public void An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file } } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file(bool compressed) + { + // Stream 5's /Length, object 4, which only its own row places, is one MaxObjectLength cuts before its value; its data + // lost its last bytes — its Flate data, its checksum and the end of its last row —, and its endstream, endobj and + // the startxref follow. The chain took the rows the data lacks for the guard's, and the Flate data run out said + // nothing, where a load of the stream, its endstream proven its own, reads the file's end. They are the file's, as + // read whole: the section is incomplete, and the index rebuilt (#224). + var file = ChainFiles.AloneWithIndirectLength(compressed: compressed, dropped: 6, lengthPadding: 1100); + using var whole = PdfDocument.Open(file); + using var document = PdfDocument.Open(file, ObjectGuard); + + whole.Reader.Structure.Sections.Should().ContainSingle().Which.Incomplete.Should().BeTrue(); + document.Reader.Structure.Sections.Should().ContainSingle().Which.Incomplete.Should().BeTrue(); + document.Reader.Structure.Sections[0].CutByLimit.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Where(code => code != PdfDiagnosticCodes.LimitObject) + .Should().Equal(whole.Diagnostics.Select(entry => entry.Code)); + document.Diagnostics.Select(entry => entry.Code).Should().Contain(PdfDiagnosticCodes.LimitObject); + + if (compressed) + { + document.Diagnostics.Should().Contain(entry => entry.Message.StartsWith("A Flate stream ends before its data does", StringComparison.Ordinal)); + } + } + [Theory] [InlineData("cut", false)] [InlineData("left out", false)] From d9f42c91f931ccd14a008648b20f5c6b426365a5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 07:22:48 +0000 Subject: [PATCH 087/146] docs: say what proves an endstream a stream's own, and what the chain reads through and keeps The pages, the architecture, ADR 34, M02 and the threat model said an endstream its object's endobj does not follow is the guard's under a hidden length. They now give the proof that makes one the stream's own: the next object the index places, the end of the file, xref or startxref, after the object's endobj or in its place; the rows a cross-reference stream lacks past such an endstream the file's. They say a section found near its named offset is the guard's as where it is named, that only a value the chain keeps loads what it read through, that /Size beside /Index is not read, which /Prev and /XRefStm the reader follows, that data whose end a guard hides decodes to the same bytes, and that an R a member's window ends on reads as the decoded data says. The rule on trailer values judges a /W element past the third as written. Refs #224, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- ...-is-readable-and-the-readers-guards-are.md | 7 +++-- docs/architecture.md | 19 ++++++++----- docs/milestones/M02.md | 15 ++++++---- docs/threat-model.md | 11 ++++---- docs/website/docs/concepts/diagnostics.md | 12 +++++--- docs/website/docs/concepts/reader-limits.md | 14 ++++++---- docs/website/docs/reference/diagnostics.md | 8 ++++-- docs/website/docs/reference/reader-limits.md | 28 +++++++++++++------ .../docs/reference/validation-rules.md | 4 +-- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 4 ++- .../Validation/PdfValidationRuleIds.cs | 2 +- 11 files changed, 78 insertions(+), 46 deletions(-) diff --git a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md index 4a967439..40ec5055 100644 --- a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md +++ b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md @@ -115,7 +115,9 @@ to an object a guard cut or left out of the index — a stream's `/Length`, `/Fi filter's parameters, a cross-reference stream's `/W`, `/Index` or `/Size`, a trailer's `/Prev` or `/XRefStm` — is the guard's, not the file's: an integer the cut parted, `1` of `15`, is only a place to look for `endstream` at, never a declared length, though a stream that lacks its `endstream` is still the file's fault, unless a header its data spells, -or an `endstream` its object's `endobj` does not follow, stops the search, the data up to it then the guard's; a whole +or an `endstream` nothing after it proves the stream's own — the next object the index places, the end of the file, +`xref` or `startxref`, after the object's `endobj` or in its place (amended on 2026-10-24) —, stops the search, the +data up to it then the guard's; a whole reference the cut left leads on, as read whole; a chain of filters ends there, its data left encoded; and a cross-reference section whose values lead to an object `MaxObjectLength` cut ends the chain, nothing rebuilt, the older sections and their trailers the guard's to hide, and the guard reached once the chain @@ -201,7 +203,8 @@ guard: active by default, reported when it is reached, and lifted by an option. whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` leads to an object `MaxObjectLength` cut, or whose `/Length` does where only it could say how far the data runs, a `/Prev` or an `/XRefStm` that does (amended on 2026-10-23) —, since an older section, or its trailer, would stand for what the cut left out; and a section a guard - cut is where it was named, not looked for near there. A rebuild that damage forces on such a chain keeps the entries + cut is where it was named, not looked for near there. One found near where it is named, whose values a guard keeps + from being read, is the guard's there too (amended on 2026-10-24). A rebuild that damage forces on such a chain keeps the entries the chain read; one the chain cannot do without — a guard that leaves it no section — runs as before. - **Or it throws.** `PdfReaderOptions.ThrowOnLimit`, false by default, turns every guard reached into a `PdfLimitExceededException`, a `PdfException` carrying the code and the name of the property: from diff --git a/docs/architecture.md b/docs/architecture.md index 86cac3fd..8fc56936 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -113,8 +113,10 @@ separate types, separate severity scales, and vocabularies that never share a co trailers, or, once the searches of a document have read four times the file, on which streams were searched first ([#138], M23). A stream whose `/Length` a guard cut or left out of the index may be valid, its length only hidden: when a header its data spells — an embedded file's does — stops its search, no `endobj` of its own before - it, or an `endstream` it spells, which its object's `endobj` does not follow, ends it, the data up to there is the - guard's, cut by it, and nothing is said of the file ([#224]). With no + it, or an `endstream` it spells ends it, the data up to there is the guard's, cut by it, and nothing is said of the + file. An `endstream` is the stream's own, and its end the file's, only where what follows the object's end — its + `endobj`, or the `endstream` where none follows — is the next object the index places, the end of the file, or + `xref` or `startxref`, read once for each place ([#224]). With no `endstream` before the stop, the object's `endobj` ends the data where it follows the declared length as an `endstream` would, or where it is the last one, a whole keyword — no regular character, solidus or percent sign right before it, which would make it part of a token, a name or a comment that @@ -134,15 +136,18 @@ separate types, separate severity scales, and vocabularies that never share a co `/DecodeParms`, a cross-reference stream's `/W`, a trailer's `/Prev` or `/XRefStm` — is not judged as one the file gave. The stream's data is found by its `endstream`, an integer the cut left only a place to look for one at, and a missing `endstream` is still the file's fault but where the search stops at a header the data - spells or an `endstream` its `endobj` does not follow (item 4); a reference the cut left whole is read through, - as a load reads it; its chain of filters ends there, with the data + spells or an `endstream` nothing after it proves the stream's own (item 4); a reference the cut left whole is read + through, as a load reads it; its chain of filters ends there, with the data left encoded; and the chain of cross-reference sections ends at that section, with nothing rebuilt and the - older sections and their trailers the guard's to hide. An object `MaxObjectLength` cut there is loaded once - the chain is read, which reaches the guard — thrown at opening under `ThrowOnLimit`. A cross-reference + older sections and their trailers the guard's to hide, wherever near its named offset the section is found. An + object `MaxObjectLength` cut there is loaded once the chain is read, which reaches the guard — thrown at opening + under `ThrowOnLimit` —, for a value the chain keeps: not for a place it dropped as no section. A cross-reference stream's `/Length` the chain could not read, which leads to such an object, ends the chain so wherever only it could say how far the data runs: with no `endstream` before the next object or the end of the file, the section is the guard's, not malformed; where the rows spell one and no `endstream` confirms what the cut left - of the length, the rows past it are the guard's. + of the length, the rows past it are the guard's — unless what follows the `endstream` the data was taken to + proves it the stream's own, as in item 4, which leaves the rows the data lacks to the file. Data whose end a + guard hides is decoded to the bytes the file gives, only what a filter says of the end silenced. ### 3.2 The write path diff --git a/docs/milestones/M02.md b/docs/milestones/M02.md index c7ad5dfc..633f8c6f 100644 --- a/docs/milestones/M02.md +++ b/docs/milestones/M02.md @@ -112,14 +112,17 @@ fault and no guard's, unless that `/Length` leads to an object a guard cut ([#22 array and dictionary an object, or a trailer, left open, and the object lacks its `endobj` ([#230]). A stream's `/Length`, `/Filter` or `/DecodeParms`, or a filter's parameter, that leads to an object a guard cut or left out of the index is the guard's, never judged as a value the file gave: the length is not judged, the data is found by -its `endstream`, and one missing is still the file's fault, unless a header the data spells, or an `endstream` its -object's `endobj` does not follow, stops the search, the data up to it then the guard's; the filters end there, the -rest of them left undone, +its `endstream`, and one missing is still the file's fault, unless a header the data spells, or an `endstream` nothing +after it proves the stream's own — the next object the index places, the end of the file, `xref` or `startxref`, after +the object's `endobj` or in its place —, stops the search, the data up to it then the guard's; the filters end there, +the rest of them left undone, `HasImageFilter` seeing no image past it, and an object stream so left is hidden as one a guard cut, `xref.checked-in-part`'s rather than `xref.object-stream-broken`'s. A cross-reference stream whose `/W`, `/Index`, -`/Size`, `/Filter` or `/DecodeParms` — or whose `/Length`, when no `endstream` ends its data — leads to an object -`MaxObjectLength` cuts while the chain is read ends the chain as a section a guard cut, as a `/Prev` or `/XRefStm` that -leads to one does, and nothing is rebuilt ([#224], [#263]). A stream's data is read each time it is asked for and kept +`/Filter` or `/DecodeParms`, or `/Size` with no `/Index` — or whose `/Length`, when no `endstream` ends its data — +leads to an object `MaxObjectLength` cuts while the chain is read ends the chain as a section a guard cut, found where +it is named or near there, as a `/Prev` or `/XRefStm` the reader follows that leads to one does, and nothing is rebuilt; +the rows its data lacks past the `endstream` such a `/Length` left it at are the guard's, unless that `endstream` is +proven the stream's own ([#224], [#263]). A stream's data is read each time it is asked for and kept by nothing — validating object streams whose data share a stretch of the file holds no copy of it for each —, and no more than `MaxStreamLength` of it is read at once, a guard reported once in the document's diagnostics and with each decoding into diagnostics a caller passes ([#170]). diff --git a/docs/threat-model.md b/docs/threat-model.md index 270a8189..c46abdf4 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -282,8 +282,8 @@ Known gaps: | A classic table of any length | Its window grows ×4 from 64 KB up to `MaxXRefSectionLength`, and the rows read are kept. A table the guard cuts, or whose rows run past a full index, keeps its trailer, found past the rows left unread by a search a megabyte at a time (`ReadTrailerPastRows`): comments are skipped, and anything else no row holds is the section's fault, reported, the search going on to the `trailer` keyword, so that its `/Encrypt` still refuses the document. The search starts at the start of the line the cut fell in, outside any comment and past the table's own keyword; it reads each megabyte once, every byte of it looked at before the next is read, however many comments or stray bytes it holds, and is not made for a place a section may have been moved to, a guess. The chain ends there, since an older section's rows would stand for the objects the unread rows define. What those rows place reads as null until the guard is raised | `ReaderLimitsTests.A_table_whose_keywords_the_bound_cuts_reports_the_bound`, `A_table_its_length_bound_cuts_keeps_its_trailer_and_the_encryption_it_names`, `A_table_its_length_bound_cuts_ends_the_chain_though_its_trailer_names_an_older_section`, `What_ends_the_rows_a_length_bound_cut_is_judged_as_it_is_where_none_did`, `A_table_a_length_bound_cuts_that_runs_to_the_end_of_the_file_has_no_trailer`, `A_table_its_length_bound_cuts_keeps_the_encryption_its_trailer_names_past_what_no_row_holds`, `A_table_cut_before_its_keyword_ends_keeps_its_trailer`, `Rows_past_a_full_index_are_not_read_and_the_trailer_past_them_is`, `A_cut_in_a_comment_among_the_rows_searches_for_the_trailer_from_the_comment_s_start`, `The_trailer_past_rows_left_unread_is_found_wherever_the_search_s_window_ends`, `A_trailer_past_rows_left_unread_that_its_own_bound_cuts_keeps_what_precedes_the_cut`; `DocumentReaderTests.The_search_for_the_trailer_past_rows_left_unread_reads_the_file_once`; `CrossReferenceChainTests.Reads_an_object_a_length_bound_cut_out_of_its_table_as_null_until_the_guard_is_lifted`, `A_section_found_near_where_prev_names_it_is_not_searched_past_the_rows_a_guard_left_unread`; `WindowEdgeTests.A_classic_table_reads_whole_wherever_its_window_edge_falls_near_its_trailer` | No: seeds stay under 120 KB | | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). A hybrid file's stream's own `/Prev`, which the chain does not follow, going on through the table's, is not read at all, nor are the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place: neither names a section missing nor ends the chain for a guard, and the rule on trailer values judges them as written, a reference they hold not followed ([#263]). An object the guard cut after a whole reference it holds hides nothing: the chain reads through it, as a load does, and loads it once the chain is read. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s`, `A_table_an_xrefstm_names_in_a_stream_s_place_has_its_own_links_left_unread`, `A_prev_written_as_a_reference_to_a_real_is_judged_through_it_only_where_the_reader_follows_it` | Yes | -| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it. Wherever the guard hides where the data ends — a guard on decoding kept it from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut is: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]) | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut` | No | +| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading, nor what it read through: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps — not for a place it dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]) | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, to the same bytes — not as data a guard cut, whose last group ASCII85 and ASCIIHex drop, which could take a decoding guard's cut away and charge the rows to the file —: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | @@ -325,7 +325,8 @@ Known gaps: | References to references that loop | `PdfReference.Resolve` gives up after 32 fetches and reads null, with no report ([#173]) | `ObjectModelTests.A_chain_of_references_that_loops_resolves_to_null` | No | | An object stream that needs an object it holds, or two that need each other | While an object stream is decoded, an object it holds reads as null and is reported once as `stream.self-reference`, at the stream's entry, or nowhere when the entry lies outside the file; the null is remembered, so a later decode reads the same | `HostileInputTests.An_object_stream_that_needs_an_object_it_holds_reads_the_same_after_the_budget_lets_it_go`, `Two_object_streams_that_need_each_other_read_the_same_after_the_budget_lets_them_go`; `HeaderOffsetTests.An_object_stream_that_needs_itself_found_near_a_row_past_the_end_is_reported_nowhere` | Object-stream seeds, not the cycle | | An offset table that lies | `/First` is checked against the decoded data, and the stream against its `/Type`, which must be `/ObjStm` ([#274]). The header is read as far as it goes, whatever `/N` claims, up to its first pair that is not two integers, that gives a number outside 1..`int.MaxValue` or an offset outside 0..`int.MaxValue`, whose offset is not past the one before it, or whose member would start at or past the end of the decoded data unless a guard cut it ([#259]) — the reader, the rebuild and the validator alike ([#202]) —: the pairs before it are kept, the fault is reported once as `object-stream.unreadable`, and the validator counts the objects the index places in the stream that the header does not list before it, and lost — none, and no finding, when it lists them all ([#275]). The header's tables grow as its pairs are read, never past the quarter of `/First`'s bytes a header holds at most, so that a `/N` no header holds allocates nothing ([#160]). A wrong index falls back on a lookup by number, sorted and built once (`ObjectStreamPlaces`), reported once for each object and definition of its stream as `object-stream.member-moved`: the object is read at the first place the header lists it. The validator's probe finds each entry's object through the same lookup, and the members written as a stream by a search, so that judging a stream costs what its members do | `ReaderTimingTests.An_object_stream_whose_every_index_is_wrong_is_read_in_linear_time`; `DocumentReaderTests.Reads_an_object_where_its_stream_s_header_lists_it_rather_than_where_the_index_says`, `Finds_an_object_its_entry_misplaces_at_the_first_place_its_object_stream_lists_it`, `Reports_an_object_stream_whose_dictionary_or_header_cannot_be_believed_where_its_data_starts`, `Reads_no_member_an_object_stream_header_numbers_as_no_object_can_be`, `Reads_no_member_an_object_stream_header_places_where_no_member_can_start`, `Ends_an_object_stream_header_at_a_member_that_starts_before_the_one_before_it_ends_or_past_the_data`, `Reads_none_of_an_object_stream_whose_first_member_starts_past_its_data`, `Reads_none_of_the_members_of_a_stream_that_is_not_of_type_object_stream`, `Reads_an_object_stream_header_that_lists_the_largest_number_a_member_can_have`, `Indexes_no_member_a_rebuild_finds_listed_under_a_number_no_object_can_have`, `Reports_an_object_at_another_index_of_its_stream_once_however_often_it_is_parsed`; `HostileInputTests.An_object_stream_that_claims_more_objects_than_it_could_hold_serves_those_it_lists_and_allocates_nothing_for_the_rest`, `An_object_stream_whose_N_its_blank_header_cannot_hold_allocates_no_table_for_it`, `An_object_stream_that_lists_one_offset_over_and_over_keeps_one_member`; `CrossReferenceRuleTests.An_object_stream_whose_header_lists_something_other_than_objects_cannot_be_read`, `An_object_stream_whose_header_breaks_after_an_object_says_which_objects_are_lost`, `An_object_stream_whose_first_member_starts_past_its_data_cannot_be_read`, `An_object_stream_that_cannot_serve_its_objects_is_reported_once`, `An_object_stream_that_lists_every_object_its_index_places_before_its_header_breaks_loses_none_and_is_no_finding`, `An_object_stream_broken_after_an_object_the_index_misplaces_shifts_that_object_and_loses_the_other`; `HostileInputTests.An_object_stream_header_s_tables_start_no_larger_than_its_bytes_can_hold`, `An_object_stream_header_s_tables_grow_no_larger_than_its_bytes_can_hold`; `ReadOrderTests.A_member_the_definition_a_rebuild_serves_lists_elsewhere_is_reported_moved_whatever_was_read_before`; `ReaderTimingTests.The_validator_judges_an_object_stream_s_members_in_linear_time` | Yes | -| A member that runs on — into the next member, or for megabytes | A member is read to the next member's start, the last to the end of the data — the offsets increase, so the members' reads together cover the data once ([#200]) —, and no further than `MaxObjectLength`, as no object of the file is ([#160]). A member left open ends there, reported as `syntax.truncated-object`, though a guard cut the data exactly at the next member's start; one the guard cut is read as far as the bound — a dictionary the bound leaves no room to say whether `stream` follows included, the bound ending after it or on a keyword that begins `stream`, or is `stream` and may be longer; a `]`, a `>>` or another keyword it ends on is whole — and reported once for each member of each definition of its stream as `limit.object`, at the stream's data, and a document opened to throw throws each time it is read | `SyntaxFaultTests.A_member_left_open_ends_at_the_next_member_s_start`; `ReaderLimitsTests.A_member_longer_than_MaxObjectLength_is_read_to_the_bound_and_each_is_reported_once`, `A_member_longer_than_MaxObjectLength_throws_each_time_it_is_read_when_the_document_throws_on_a_limit`, `A_member_within_MaxObjectLength_is_read_whole_and_reported_nowhere`; `HostileInputTests.A_member_of_millions_of_arrays_is_read_no_further_than_MaxObjectLength`, `An_object_stream_of_members_one_byte_apart_reads_each_to_the_next_one_s_start_in_time`; `ReaderTimingTests.The_members_of_an_object_stream_one_byte_apart_are_read_in_linear_time`; `ReaderLimitsTests.A_member_dictionary_whose_stream_keyword_MaxObjectLength_leaves_out_is_cut_there_and_reported`, `A_member_whose_value_ends_where_MaxObjectLength_ends_its_window_is_read_whole`, `A_member_dictionary_followed_by_a_keyword_that_cannot_be_stream_where_MaxObjectLength_ends_its_window_is_whole`, `A_member_whose_array_ends_where_a_guard_cuts_the_data_is_read_whole`, `A_member_left_open_where_a_guard_cuts_the_data_at_the_next_member_s_start_is_the_file_s_fault` | Yes | +| A member that runs on — into the next member, or for megabytes | A member is read to the next member's start, the last to the end of the data — the offsets increase, so the members' reads together cover the data once ([#200]) —, and no further than `MaxObjectLength`, as no object of the file is ([#160]). A member left open ends there, reported as `syntax.truncated-object`, though a guard cut the data exactly at the next member's start; one the guard cut is read as far as the bound — a dictionary the bound leaves no room to say whether `stream` follows included, the bound ending after it or on a keyword that begins `stream`, or is `stream` and may be longer; a `]`, a `>>` or another keyword it ends on is whole, an `R` too where the decoded +byte past the window does not go on it, `6 0 R` read as the reference and `6 0 RG` cut ([#224]) — and reported once for each member of each definition of its stream as `limit.object`, at the stream's data, and a document opened to throw throws each time it is read | `SyntaxFaultTests.A_member_left_open_ends_at_the_next_member_s_start`; `ReaderLimitsTests.A_member_longer_than_MaxObjectLength_is_read_to_the_bound_and_each_is_reported_once`, `A_member_longer_than_MaxObjectLength_throws_each_time_it_is_read_when_the_document_throws_on_a_limit`, `A_member_within_MaxObjectLength_is_read_whole_and_reported_nowhere`; `HostileInputTests.A_member_of_millions_of_arrays_is_read_no_further_than_MaxObjectLength`, `An_object_stream_of_members_one_byte_apart_reads_each_to_the_next_one_s_start_in_time`; `ReaderTimingTests.The_members_of_an_object_stream_one_byte_apart_are_read_in_linear_time`; `ReaderLimitsTests.A_member_dictionary_whose_stream_keyword_MaxObjectLength_leaves_out_is_cut_there_and_reported`, `A_member_whose_value_ends_where_MaxObjectLength_ends_its_window_is_read_whole`, `A_member_whose_reference_ends_where_MaxObjectLength_ends_its_window_is_the_reference`, `A_member_whose_R_the_byte_past_MaxObjectLength_continues_is_cut`, `A_member_dictionary_followed_by_a_keyword_that_cannot_be_stream_where_MaxObjectLength_ends_its_window_is_whole`, `A_member_whose_array_ends_where_a_guard_cuts_the_data_is_read_whole`, `A_member_left_open_where_a_guard_cuts_the_data_at_the_next_member_s_start_is_the_file_s_fault` | Yes | | A stream written inside an object stream, which ISO 32000 forbids | The parser stops at its `stream` keyword: the object reads as null, nothing of it is decoded nor kept, and it is reported once for each definition of its stream and number as `object-stream.member-stream`; the validator reports the entry that places it there as `xref.entry-broken`, naming the index its header gives when the entry gives another, and the entry of the index a rebuild made where the file's own places nothing — or, where a guard stopped the chain, the part of it the reader read ([#161]) | `DocumentReaderTests.Reads_an_object_written_as_a_stream_inside_an_object_stream_as_null_and_reports_it_once`; `HostileInputTests.A_stream_inside_an_object_stream_is_not_decoded_whatever_it_would_decode_to`; `HostileInputMemoryTests.Object_streams_each_holding_a_stream_keep_no_more_than_their_budget_once_their_members_are_read`; `StreamLengthTests.A_stream_inside_an_object_stream_has_no_length_read_nor_reported_however_often_it_is_read`; `CrossReferenceRuleTests.An_object_written_as_a_stream_inside_an_object_stream_is_broken`, `An_object_whose_bytes_merely_hold_stream_inside_an_object_stream_is_sound`, `An_object_written_as_a_stream_that_its_entry_misplaces_is_broken_where_its_header_lists_it`, `An_object_written_as_a_stream_that_only_a_rebuilt_index_places_is_broken`, `An_object_written_as_a_stream_that_the_chain_and_a_rebuilt_index_both_place_is_its_entry_s_finding_alone`; `ReadOrderTests.A_member_fault_of_the_definition_a_rebuild_serves_is_reported_whatever_was_read_before` | Object-stream seeds, not a stream member | | Many large object streams held at once | `ObjectStreamBudget` (32 MB, with its reason), which weighs each stream's decoded data with its header's tables and lookup ([#160]): the oldest are let go and decoded again on demand, and nothing a member gave keeps its stream's data alive past them ([#161]) | `HostileInputTests.Object_streams_are_kept_decoded_within_a_budget_and_decoded_again_when_needed`, `The_object_stream_budget_weighs_the_header_tables_with_the_data`; `HostileInputMemoryTests.Disposing_a_document_lets_go_of_the_object_streams_it_decoded_though_a_reference_it_gave_is_held` | No | | A stream decoded again and again, each decoding reporting into its document | Its document's diagnostics hold what decoding a stream met once — a filter's fault, a guard reached on its data —: a decoding reports apart, and a report the document holds already, the whole of it alike, is not moved in, so that it takes no room there; two filters of one decoding that report alike are each held. An object stream decoded again after the budget let it go, by a rebuild, by the validator or by a caller adds nothing it holds already. The guard is still counted, so that what it cut is never taken for damage, and a document opened to throw throws each time, what the decoding met before moved in all the same ([#144]) | `HostileInputTests.A_fault_of_an_object_stream_is_held_once_by_its_document_however_often_the_stream_is_decoded`, `A_guard_an_object_stream_reaches_is_held_once_by_its_document_and_counted_each_time`, `A_document_holds_alike_reports_of_two_streams_and_of_two_filters_of_one_stream_each_once_a_decoding`, `What_a_decoding_met_before_a_guard_threw_is_held_once_though_each_decoding_throws`, `A_report_its_document_holds_takes_no_room_when_a_stream_is_decoded_again_into_full_diagnostics`, `A_report_the_capacity_leaves_out_of_one_decoding_is_counted_by_its_document`, `A_report_full_diagnostics_left_out_is_counted_again_each_time_its_stream_is_decoded`; `DiagnosticsTests.A_copy_of_a_report_the_capacity_left_out_is_counted_again_each_time_a_decoding_meets_it`, `A_decoding_moves_in_only_what_no_decoding_moved_in_before_two_alike_of_its_own_included`, `A_report_a_decoding_moved_in_before_takes_no_room_once_the_diagnostics_are_full_and_a_guard_stays_counted`; `CorpusReadingTests.A_document_holds_each_report_once_however_often_its_streams_are_decoded`, `A_damaged_object_stream_s_fault_is_held_once_after_a_full_read_and_a_validation` | No | @@ -348,12 +349,12 @@ Known gaps: | Threat | Defense | Held by | Fuzzed | |---|---|---|---| | A `/Length` that is negative, past `int.MaxValue`, not an integer or unresolvable | Only 0..`int.MaxValue` is taken as a length. Anything else sends the reader to the one search for the stream's first `endstream` (below) | `StreamLengthTests.A_length_that_cannot_be_taken_is_reported_as_the_file_wrote_it` | Partly | -| A `/Length` naming an object a guard cut or left out of the index, judged by what the cut left of it: null, reported as no length, or the `1` it parted of `15`, reported as a length the data contradicts; or the stream's missing `endstream` left unreported along with the length | Keyed on the object the reference leads to, through a chain of references too, never on the value read (`StreamLengthForm.ObjectCut`, `ObjectUnindexed`): an integer read before the cut is only a place to look for `endstream` at, never kept as a length, and the data is otherwise found by its `endstream`. Nothing is said of the length, whose fault is the guard's, reported where it was reached; a missing `endstream` is still the file's, and reported — but where the search stops at a header the data spells, or at an `endstream` its object's `endobj` does not follow (below) —, naming the object at the end of the chain of references, the one the guard cut or left out: "The stream's /Length names object N G, which one of the reader's limits cut, and no endstream follows its data …", or "…, which one of the reader's limits left out of the index, and no endstream follows its data …". A cross-reference stream's `/Length` so cut is left alike once the chain is read, and makes its section the guard's when no `endstream` ends its data, as *Indexing* says ([#224]) | `GuardCutValueTests.A_length_a_guard_cut_is_not_judged_and_the_data_is_found_by_its_endstream`, `A_length_a_guard_cut_leaves_a_missing_endstream_the_file_s_fault`, `A_length_an_object_stream_s_guard_left_unread_is_not_judged`, `A_length_whose_reference_leads_to_an_object_a_guard_left_out_of_the_index_is_not_judged`, `A_missing_endstream_names_the_object_a_guard_cut_at_the_end_of_the_length_s_chain_and_keeps_no_length`, `A_missing_endstream_is_the_file_s_fault_when_a_guard_left_the_length_out_of_the_index`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`; `GuardCutValueTests.A_length_whose_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `A_missing_endstream_names_the_last_object_of_a_length_s_chain_a_guard_cut_or_left_out_of_the_index`, `A_missing_endstream_no_header_stops_the_search_at_is_the_file_s_where_a_guard_hid_the_length` | No | +| A `/Length` naming an object a guard cut or left out of the index, judged by what the cut left of it: null, reported as no length, or the `1` it parted of `15`, reported as a length the data contradicts; or the stream's missing `endstream` left unreported along with the length | Keyed on the object the reference leads to, through a chain of references too, never on the value read (`StreamLengthForm.ObjectCut`, `ObjectUnindexed`): an integer read before the cut is only a place to look for `endstream` at, never kept as a length, and the data is otherwise found by its `endstream`. Nothing is said of the length, whose fault is the guard's, reported where it was reached; a missing `endstream` is still the file's, and reported — but where the search stops at a header the data spells, or at an `endstream` nothing after it proves the stream's own (below) —, naming the object at the end of the chain of references, the one the guard cut or left out: "The stream's /Length names object N G, which one of the reader's limits cut, and no endstream follows its data …", or "…, which one of the reader's limits left out of the index, and no endstream follows its data …". A cross-reference stream's `/Length` so cut is left alike once the chain is read, and makes its section the guard's when no `endstream` ends its data, as *Indexing* says ([#224]) | `GuardCutValueTests.A_length_a_guard_cut_is_not_judged_and_the_data_is_found_by_its_endstream`, `A_length_a_guard_cut_leaves_a_missing_endstream_the_file_s_fault`, `A_length_an_object_stream_s_guard_left_unread_is_not_judged`, `A_length_whose_reference_leads_to_an_object_a_guard_left_out_of_the_index_is_not_judged`, `A_missing_endstream_names_the_object_a_guard_cut_at_the_end_of_the_length_s_chain_and_keeps_no_length`, `A_missing_endstream_is_the_file_s_fault_when_a_guard_left_the_length_out_of_the_index`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`; `GuardCutValueTests.A_length_whose_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `A_missing_endstream_names_the_last_object_of_a_length_s_chain_a_guard_cut_or_left_out_of_the_index`, `A_missing_endstream_no_header_stops_the_search_at_is_the_file_s_where_a_guard_hid_the_length` | No | | An indirect `/Length` leading back to its stream, or chained | The loading set and `MaxNestedLoads`, as above | `HostileInputTests.Survives_an_object_whose_length_refers_to_itself`, `Survives_a_chain_of_lengths_longer_than_the_stack_is_deep` | No | | A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | -| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say — that the object's `endobj` does not follow, which only the hidden length could tell from the stream's own, a reading that cannot tell where the object ends taking back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | +| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places, the end of the file, or `xref` or `startxref`, which data that goes on is never followed by: read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | | Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | diff --git a/docs/website/docs/concepts/diagnostics.md b/docs/website/docs/concepts/diagnostics.md index dd56f77d..41aea26f 100644 --- a/docs/website/docs/concepts/diagnostics.md +++ b/docs/website/docs/concepts/diagnostics.md @@ -71,9 +71,12 @@ the index is not judged either: its data is found the same way, and a missing `e stream may be valid, its length only hidden, and its data may spell an object header, as an embedded PDF file's does — and the `endstream` of a stream the embedded file holds, which no search can tell from the stream's own. A header the data spells that stops the search, with no `endobj` of the object's own before it, leaves the data up to it to the -guard, and so does an `endstream` the data spells — inside a string, say — that the object's `endobj` does not -follow: the data is marked cut by the guard, whose report the document holds, whether the object ends with `endobj` is -not judged, and nothing is said of the file. A whole reference the cut left, `6 0 R` before the window's end, is no length +guard, and so does an `endstream` the data spells — inside a string, say, an `endobj` after it or not —: the data is +marked cut by the guard, whose report the document holds, whether the object ends with `endobj` is not judged, and +nothing is said of the file. Only what follows the object's end — its `endobj`, or the `endstream` where none follows — +can prove an `endstream` the stream's own: the next object the index places, the end of the file, or the `xref` or +`startxref` keyword, which data that goes on is never followed by. The end is then the file's, and so is an `endobj` +the object lacks. A whole reference the cut left, `6 0 R` before the window's end, is no length it hid: it leads on to the object it names, as read whole. A cross-reference stream whose `/Length` the chain cannot read yet, which a valid file may hold, is searched as well, up @@ -82,7 +85,8 @@ rows may spell an object header and end in a carriage return: no header stops it the line feed that precedes `endstream` is kept. Data with no `endstream` before that stop ends nowhere the section says, and its rows are not read — the file's fault, unless the `/Length` leads to an object one of the reader's limits cut: the length that would have said where the data ends is then the guard's, and so is the section, the chain ending -there. +there. Where such a `/Length` leaves the data at the first `endstream`, the rows it lacks are the guard's too, unless +what follows that `endstream` proves it the stream's own, as above: they are then the file's to lack. The search is not one of the reader's limits: a valid file's `endstream` follows its length, so only a damaged stream is searched — and a cross-reference stream whose `/Length` the chain cannot read yet, and a stream whose `/Length` a guard diff --git a/docs/website/docs/concepts/reader-limits.md b/docs/website/docs/concepts/reader-limits.md index db1ac127..685a26fa 100644 --- a/docs/website/docs/concepts/reader-limits.md +++ b/docs/website/docs/concepts/reader-limits.md @@ -53,12 +53,14 @@ A value read through a reference to an object a guard cut, or left out of the in left of it may be the start of what the file holds — `1` of `15`, `/Flate` of `/FlateDecode` —, so it is never judged as a value the file gave; a whole reference it left, though, leads on to the object it names, as read whole. The data of a stream whose `/Length` is so named is found by its `endstream`, and its length is not judged, though an `endstream` -the data lacks is still the file's to lack — unless an object header the data spells, or an `endstream` its object's -`endobj` does not follow, stops the search: only the hidden length could tell those from the end of the data, which is -then the guard's too; decoding stops at a filter so named, or -one whose parameters are, and leaves the rest of the data encoded, as past the most filters the reader runs; and a -`/Prev` or an `/XRefStm` so named, or a cross-reference stream whose values lead to an object a guard cut, ends the -chain as a section a guard cut does. +the data lacks is still the file's to lack — unless an object header the data spells stops the search, or an +`endstream` it spells, which only the hidden length could tell from the end of the data: the data is then the guard's +too. An `endstream` is the stream's own beyond doubt only where what follows its object's end is something data that +goes on cannot be followed by — the next object the index places, the end of the file, a cross-reference section's +keyword —, and the end is then the file's. Decoding stops at a filter so named, or one whose parameters are, and leaves +the rest of the data encoded, as past the most filters the reader runs; and a `/Prev` or an `/XRefStm` the reader +follows that is so named, or a cross-reference stream whose values lead to an object a guard cut, ends the chain as a +section a guard cut does, wherever near its named offset the section is found. What a guard leaves out of the cross-reference index — the oldest sections of a long chain, the rows past a long table's bound, the objects past the most the index holds — is no damage either. Those objects read as null, unknown diff --git a/docs/website/docs/reference/diagnostics.md b/docs/website/docs/reference/diagnostics.md index 252d6cf5..5ca4f961 100644 --- a/docs/website/docs/reference/diagnostics.md +++ b/docs/website/docs/reference/diagnostics.md @@ -186,9 +186,11 @@ judged either: the data is found by its `endstream` the same way, and a missing object the guard left out — `The stream's /Length names object 21 0, which one of the reader's limits left out of the index, and no endstream follows its data`, then where the search stopped and what was taken. For either, an object header the data spells, as an embedded PDF file's does, that stops the search — no `endobj` of the object's own before -it —, or an `endstream` the data spells that the object's `endobj` does not follow, leaves the data up to it to the -guard: nothing is reported of the stream, whose data is cut by the guard, and whether its object ends with `endobj` is -not judged. +it —, or an `endstream` the data spells, an `endobj` after it or not, leaves the data up to it to the guard: nothing is +reported of the stream, whose data is cut by the guard, and whether its object ends with `endobj` is not judged. An +`endstream` is taken for the stream's own, its end the file's, only where what follows its object's end — its +`endobj`, or the `endstream` where none follows — is the next object the index places, the end of the file, or the +`xref` or `startxref` keyword. ```text Warning stream.length-invalid at 5501: The stream declared 19954 bytes but ended after 19952. diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index 95e41f0e..4f8d8c72 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -113,10 +113,13 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. of a longer one, is only a place to look for `endstream` at. No `endstream` after the data of a stream whose `/Length` names an object a guard cut, or left out of the index, is still the file's fault, and reported; the report names the object at the end of the chain of references. The search may stop short of it, though, at an object header - the data spells, as an embedded PDF file's does, or at an `endstream` the object's `endobj` does not follow, as a - string may spell one: the length that tells those from the end of the data is the guard's, so the data up to there - comes back cut by the guard, `CutByGuard`, whether the object ends with `endobj` is not judged, and nothing is said of - the file. A reference the cut left whole, the cut falling after `6 0 R`, hides nothing: it is read through to the + the data spells, as an embedded PDF file's does, or at an `endstream` the data spells, as a string may, an `endobj` + after it or not: the length that tells those from the end of the data is the guard's. An `endstream` ends data that + is the file's only where what follows its object's end — its `endobj`, or the `endstream` itself where none follows — + proves it the stream's own: the next object the index places, the end of the file, or the `xref` or `startxref` + keyword, which no data that goes on can be followed by. Elsewhere the data up to there comes back cut by the guard, + `CutByGuard`, whether the object ends with `endobj` is not judged, and nothing is said of the file. A reference the cut + left whole, the cut falling after `6 0 R`, hides nothing: it is read through to the object it names, as it is read whole, while the chain is read as afterward. Decoding stops at a filter so named, or at the filter whose parameters are: the filters before it run, and the stream comes back as they decoded it, still encoded by the rest, @@ -124,11 +127,18 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. a Flate or LZW filter that comes before it included, and says false when such a value ends the chain there; under `ThrowOnLimit`, loading one throws where decoding would. An object stream the reader meets so is read as one a guard cut before its data: its objects are unknown until you raise the limit, and when the catalog - is among them the reader does not look for another. While the chain is read, a `/Prev` or an `/XRefStm` that leads to - an object `MaxObjectLength` cuts, a cross-reference stream whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` - does, and one whose `/Length` does when no `endstream` ends its data, are the guard's, as a section it cut is: the - chain ends there, the sections it names are not read — the older ones, or a hybrid file's cross-reference stream —, - the index is not rebuilt, and the guard is reported once the chain is read, or thrown by `Open`. + is among them the reader does not look for another. While the chain is read, a `/Prev` or an `/XRefStm` the reader + follows that leads to an object `MaxObjectLength` cuts — not a hybrid file's cross-reference stream's own `/Prev`, + nor the links of a table an `/XRefStm` names in a stream's place, which it leaves unread —, a cross-reference stream + whose `/W`, `/Index`, `/Filter` or `/DecodeParms` does, or whose `/Size` does with no `/Index`, and one whose + `/Length` does when no `endstream` ends its data, are the guard's, as a section it cut is: the chain ends there, the + sections it names are not read — the older ones, or a hybrid file's cross-reference stream —, the index is not + rebuilt, and the guard is reported once the chain is read, or thrown by `Open`. So does a section found near the + offset it is named at, which such a value keeps from being read. The rows a cross-reference stream's data lacks, + where its `/Length` so leads and its data was taken to the first `endstream`, are the guard's too, unless what follows + that `endstream` proves it the stream's own, as above: they are then the file's to lack, and the index is rebuilt. An + object the chain read through a reference the cut left whole is loaded once the chain is read only for a value it + keeps: not for a place it dropped as no section. - A stream read from a document decodes under **that document's** limits, however long after opening you decode it. A stream you build in memory decodes under `PdfReaderLimits.Default`. - A classic trailer that ends within the window its cross-reference table was read through is read whole, whatever diff --git a/docs/website/docs/reference/validation-rules.md b/docs/website/docs/reference/validation-rules.md index a19f1d26..2e5bad05 100644 --- a/docs/website/docs/reference/validation-rules.md +++ b/docs/website/docs/reference/validation-rules.md @@ -45,7 +45,7 @@ References are to ISO 32000-1:2008. | `file.trailer-malformed` | Error | A trailer is not a well-formed dictionary: the `trailer` keyword is not followed by one, or the reader read it despite syntax errors — a key that is not a name, a dictionary or a string left unclosed, a hexadecimal string holding bytes that are neither digits nor white space, a name whose number sign is no escape, a key given twice —, so an entry may have been lost with the syntax, or read otherwise than its writer meant. A cross-reference stream's dictionary is held to the same. | Each trailer of the chain, as the reader parsed it | | `file.root-invalid` | Error | The trailer's `/Root` is missing or does not lead to a document catalog (Table 15), a value that is not a reference included — unless it is a catalog written in the trailer itself, which the reader takes as it is ([#111]). The reader then looks for an object of `/Type /Catalog` among those the index holds — and rebuilds the index only when none is one —: the catalog it finds is its choice, not the file's. Not judged when the trailer could not be read at all, nor when the index was rebuilt as the document opened, nor when one of the reader's limits left the catalog out of the index as the document opened, which `xref.checked-in-part` says, or cut the object `/Root` names — through a chain of references too, and in an object stream whose cut leaves it unread, past the cut data or in a stream the guard made unreadable ([#259]), or in one whose filters end at a `/Filter`, a `/DecodeParms` or a parameter that leads to an object a guard cut or left out of the index, which leaves its data encoded ([#263]) — or the trailer a catalog written there was read from, which the reader then takes for the catalog as it read it, looking for no other, and its `limit.*` report names ([#175]) — whatever a rebuild found since —, nor a `/Root` the trailers lack where one of the reader's limits kept a trailer from being read whole — cut, or older than where the chain stopped, at the section count or at a section whose rows a limit cut or filled the index with —: it may lie there, and the reader rebuilds nothing for it. Rows a limit cut that run on to no trailer lack one: the file's, which a rebuild answers. The catalog the finding names is the one the reader took last, in the rebuild run before the first rule if there was one. | The trailer the chain gave, merged as readers merge it, and `/Root` before the reader looked for a catalog | | `file.size-wrong` | Warning | A section's `/Size` is missing, or is not one more than the highest object number it and the sections it updates use (Tables 15 and 17, 7.5.6, 7.5.8.4). A hybrid file's cross-reference stream meets one of Table 17's two readings or the other — one more than the highest number it and the sections it updates use, or the `/Size` of the table that names it —, and either is accepted. A `/Size` that leaves in-use objects numbered above it is `xref.object-past-size`'s. A `/Size` written as a reference, or as a real, is said to be. Not judged when the chain lost a section, or one of the reader's limits cut it or ended it before the sections a `/Prev` or an `/XRefStm` names ([#263]). | Each section's trailer, and the highest number each section indexes | -| `file.trailer-value-wrong` | Warning | A value a section's trailer gives the reader to read the chain is written as the specification does not allow, and read all the same, as other readers read it: a reference where it must be direct, read where a section read before it places the object; a real with no fractional part where it must be an integer, read as the integer it equals ([#182], [#215]). A cross-reference stream's `/Type`, `/Index`, `/Prev` and `/W`, the elements of `/Index` and `/W`, and its `/Filter` and `/DecodeParms` are direct (7.5.8.2); in a file that declares PDF 2.0, so are its `/Length` and the elements and entries of its `/Filter` and `/DecodeParms` (an erratum of ISO 32000-2, pdf-issues #246). A classic trailer's `/Prev` is direct (Table 15); an `/XRefStm` may be indirect (Table 19). A value the reader does not read is judged as the file wrote it, a reference it holds not followed, which would read what the reading never did: a hybrid file's cross-reference stream's own `/Prev`, which the chain does not follow, going on through the table's; the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place; and what of a cross-reference stream's `/Filter` and `/DecodeParms` decoding does not read — the parameters of a filter that reads none, `/Colors`, `/BitsPerComponent` and `/Columns` where no predictor is, `/EarlyChange` but for LZW, or what lies past the element that ends the chain ([#263]). `/W`, `/Index`, `/Prev`, `/XRefStm`, a stream's `/Length` and its `/Predictor`, `/Colors`, `/BitsPerComponent`, `/Columns` and `/EarlyChange` are integers. One finding per value, which says both faults of a reference that leads to a real where the reader follows it. A value the chain is read with that the reader could not read so is the section's fault — `xref.section-malformed`, `xref.section-not-found` —, unless it leads to an object one of the reader's limits cut, where the chain ends for the limit, a section whose rows need it left to `xref.checked-in-part` ([#263]), and `/Size` is `file.size-wrong`'s; a `/Type`, or a `/Length` read once the chain is, is reported here whether or not it could be read. | Each section's trailer, as the file wrote it, and the objects the references the reader follows name, through the index the reader settled on before the first rule | +| `file.trailer-value-wrong` | Warning | A value a section's trailer gives the reader to read the chain is written as the specification does not allow, and read all the same, as other readers read it: a reference where it must be direct, read where a section read before it places the object; a real with no fractional part where it must be an integer, read as the integer it equals ([#182], [#215]). A cross-reference stream's `/Type`, `/Index`, `/Prev` and `/W`, the elements of `/Index` and `/W`, and its `/Filter` and `/DecodeParms` are direct (7.5.8.2); in a file that declares PDF 2.0, so are its `/Length` and the elements and entries of its `/Filter` and `/DecodeParms` (an erratum of ISO 32000-2, pdf-issues #246). A classic trailer's `/Prev` is direct (Table 15); an `/XRefStm` may be indirect (Table 19). A value the reader does not read is judged as the file wrote it, a reference it holds not followed, which would read what the reading never did: a hybrid file's cross-reference stream's own `/Prev`, which the chain does not follow, going on through the table's; the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place; the elements of a `/W` past the third, the widths of three fields being all the reader reads; and what of a cross-reference stream's `/Filter` and `/DecodeParms` decoding does not read — the parameters of a filter that reads none, `/Colors`, `/BitsPerComponent` and `/Columns` where no predictor is, `/EarlyChange` but for LZW, or what lies past the element that ends the chain, under the reader's `MaxFilterChainLength` ([#263]). What decoding reads is judged as it read it, whatever the object cache let go since. `/W`, `/Index`, `/Prev`, `/XRefStm`, a stream's `/Length` and its `/Predictor`, `/Colors`, `/BitsPerComponent`, `/Columns` and `/EarlyChange` are integers. One finding per value, which says both faults of a reference that leads to a real where the reader follows it. A value the chain is read with that the reader could not read so is the section's fault — `xref.section-malformed`, `xref.section-not-found` —, unless it leads to an object one of the reader's limits cut, where the chain ends for the limit, a section whose rows need it left to `xref.checked-in-part` ([#263]), and `/Size` is `file.size-wrong`'s; a `/Type`, or a `/Length` read once the chain is, is reported here whether or not it could be read. | Each section's trailer, as the file wrote it, and the objects the references the reader follows name, through the index the reader settled on before the first rule | ### Cross-references @@ -87,7 +87,7 @@ it is checked: the exception above judges the index the reader rebuilt, and the | Rule | Severity | Meaning | Reads | |---|---|---|---| | `object.reference-missing` | Warning | An object reachable from the trailer refers to an object the file does not define — none in its index, a free entry, or object 0, which heads the free list and is never in use (7.5.4) —, which is null (7.3.10): the reader reads it so, rebuilding nothing, and what the reference was to give is lost. One finding per object that holds such references, naming the first. An object one of the reader's limits left out of the index is unknown rather than missing, and is `xref.checked-in-part`'s. An object the index holds and the reader cannot produce is `xref.entry-broken`'s or `xref.object-stream-broken`'s — but one a rebuild serves from another object stream than the chain's index places it in, or from another definition of that stream, which does not list it or holds it written as a stream, and so loses, which no finding names yet ([#276]) —, a kid of the page tree `page-tree.kid-invalid`'s — one in a node's `/Kids`, an array of kids of its own or a root written in the catalog the walk read whole, not what an array written among the kids holds, which the walk does not read —, and a trailer's `/Root` that names nothing `file.root-invalid`'s. | Every object reachable from the trailer, once each | -| `object.endobj-missing` | Warning | An object reachable from the trailer does not end with `endobj` (7.3.10): another token, or the end of the file, follows its value. The reader reads the value as far as it goes, as qpdf does while reporting it. An empty object, `2 0 obj endobj`, has its `endobj`. An object whose array or dictionary was never closed lacks it where the next object's header — `N G obj` in digits, white space alone between them, an object number from 1 to 2,147,483,647 and a generation up to 65,535 —, met where an element, a key or a value is expected, ends every container left open ([#230]); an array or a dictionary that holds such a header and is closed after it ends there all the same, its `endobj` judged missing, which none of the corpus's 344,361 objects does. What follows a stream's `endstream` is read through white space and comments however long, and the file asked for it when it lies past the 8 KB window the reader first reads through — after a stream whose data runs past that window ([#55]), or an `endstream` the window's edge comes too soon after ([#174]). A stream with no `endstream` has its `endobj` when the object's `endobj` follows the length it declared — after white space and comments, a comment only once white space comes before it —, or ends its data with nothing but white space and comments after it before the next object or the end of the file; one whose data runs on to the next object, or to the end of the file, lacks it. Where the object ends is unknown, and is not judged, after a stream whose declared length the reader kept with neither an `endstream` nor an `endobj` after it, one whose data, run on to the next object or the end of the file, holds an `endobj` that may be its object's own, and one whose end lies past what the reader reads first once the document's searches for `endstream` have read as much of the file as they may. So is a stream whose `/Length` one of the reader's limits cut or left out of the index, when an object header its data spells, or an `endstream` it spells that the object's `endobj` does not follow, ends its data: only that length could tell either from the stream's own end ([#224]). | What follows each reachable object's value, as the reader read it | +| `object.endobj-missing` | Warning | An object reachable from the trailer does not end with `endobj` (7.3.10): another token, or the end of the file, follows its value. The reader reads the value as far as it goes, as qpdf does while reporting it. An empty object, `2 0 obj endobj`, has its `endobj`. An object whose array or dictionary was never closed lacks it where the next object's header — `N G obj` in digits, white space alone between them, an object number from 1 to 2,147,483,647 and a generation up to 65,535 —, met where an element, a key or a value is expected, ends every container left open ([#230]); an array or a dictionary that holds such a header and is closed after it ends there all the same, its `endobj` judged missing, which none of the corpus's 344,361 objects does. What follows a stream's `endstream` is read through white space and comments however long, and the file asked for it when it lies past the 8 KB window the reader first reads through — after a stream whose data runs past that window ([#55]), or an `endstream` the window's edge comes too soon after ([#174]). A stream with no `endstream` has its `endobj` when the object's `endobj` follows the length it declared — after white space and comments, a comment only once white space comes before it —, or ends its data with nothing but white space and comments after it before the next object or the end of the file; one whose data runs on to the next object, or to the end of the file, lacks it. Where the object ends is unknown, and is not judged, after a stream whose declared length the reader kept with neither an `endstream` nor an `endobj` after it, one whose data, run on to the next object or the end of the file, holds an `endobj` that may be its object's own, and one whose end lies past what the reader reads first once the document's searches for `endstream` have read as much of the file as they may. So is a stream whose `/Length` one of the reader's limits cut or left out of the index, when an object header its data spells, or an `endstream` it spells, ends its data: only that length could tell either from the stream's own end. An `endstream` is the stream's own, and an `endobj` its object lacks judged missing, only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places, the end of the file, or the `xref` or `startxref` keyword ([#224]). | What follows each reachable object's value, as the reader read it | | `object.name-null-character` | Warning | An object reachable from the trailer holds a name with a null character, written `#00`, which a name cannot contain (7.3.5). The reader keeps the name; qpdf refuses it and reads dictionaries without the keys so named. One finding per object, naming the first. | Every name of every reachable object | | `object.key-missing` | Warning | An object reachable from the trailer lacks a key the [Arlington PDF Model](#object-shapes-from-the-arlington-pdf-model) requires of its type — absent, or `null`, which in a dictionary is the same (7.3.7): a key the model requires once the version the file declares has it, or one it requires in some versions only, such as a form XObject's `/Name` in PDF 1.0. An inheritable key is looked for up the `/Parent` chain, and a field's `/DA` in the interactive form's too; a key only an extension of ISO 32000 defines is never required. An array shorter than the elements it requires lacks each required element it does not hold, one finding per element. [One finding per type and key](#object-shapes-from-the-arlington-pdf-model), at the first object, with how many lack it. | Every object reachable from the trailer, typed along the model's links, once each | | `object.value-type-wrong` | Warning | A value is of a type the Arlington PDF Model does not allow for its key, judged by the value's class: a real, even `400.0`, where an integer belongs; a string where a name does; a stream where a dictionary does, or the reverse, which is reported on the object that holds it and not walked into. A `null` written in an array is judged like any other value; one reached through a reference is not judged. One finding per type and key. | As `object.key-missing` | diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 01ed2c65..c0ab1942 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -728,7 +728,9 @@ public PdfObject GetObject(PdfObjectId id) /// left unread, and the guard is neither reported nor counted: the chain's sections are cut only by their own windows /// and decoding, and a guard that holds is reached when the object is loaded once the chain is read. One whose value /// the cut left a whole reference reads as that reference, as a load reads through it, and is loaded once the chain is - /// read for its guard to be reached. The objects read through it nest as loads do, within . + /// read for its guard to be reached — unless what read it was a place the section may have been moved to, which the + /// chain dropped (). The objects read through it nest as loads do, within + /// . /// /// private PdfObject? ReadWithoutLoading(PdfObjectId id) diff --git a/src/AdCodicem.Pdf/Validation/PdfValidationRuleIds.cs b/src/AdCodicem.Pdf/Validation/PdfValidationRuleIds.cs index f7d74f0d..9d3427bd 100644 --- a/src/AdCodicem.Pdf/Validation/PdfValidationRuleIds.cs +++ b/src/AdCodicem.Pdf/Validation/PdfValidationRuleIds.cs @@ -74,7 +74,7 @@ public static class PdfValidationRuleIds /// A value a section's trailer — a classic table's, or a cross-reference stream's dictionary — gives the reader to read /// the chain is written as the specification does not allow: a reference where it must be direct, a real with no /// fractional part where it must be an integer. The reader reads such a value all the same, or does not read it at all - /// — a hybrid file's stream's own /Prev —, which is then judged as written. + /// — a hybrid file's stream's own /Prev, a /W element past the third —, which is then judged as written. /// . /// public const string FileTrailerValueWrong = "file.trailer-value-wrong"; From cea1c7bfff5c2376550b908e73bf337d86427456 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 10:06:21 +0000 Subject: [PATCH 088/146] fix: decode data whose end a guard hides as cut data, counting a dropped group against the bound Round 6 decoded such data to the bytes its first decode gave, so that the guard decision could not flip; but those bytes guessed what the hidden end parted: an ASCII85 group completed with u's, which could carry into a row's generation, and a predicted row left as decoded, its tag read as a row's type, which freed the catalog and rebuilt the index of a valid file. The data decodes again as data a guard cut, the parted group dropped and the predicted row undone as far as it goes, and ASCII85 and ASCIIHex count a group they drop against MaxDecodedStreamLength, the whole data holding at least its bytes: the bound is reached, or not, as the first decode found. Refs #220, #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/Filters/Ascii85Filter.cs | 12 +- .../IO/Filters/AsciiHexFilter.cs | 12 +- .../IO/Filters/PdfFilterPipeline.cs | 62 ++++---- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 133 ++++++++++++++++++ tests/AdCodicem.Pdf.Tests/FilterTests.cs | 29 ++++ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 50 +++++++ 6 files changed, 258 insertions(+), 40 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/Filters/Ascii85Filter.cs b/src/AdCodicem.Pdf/IO/Filters/Ascii85Filter.cs index 06fa3629..32317651 100644 --- a/src/AdCodicem.Pdf/IO/Filters/Ascii85Filter.cs +++ b/src/AdCodicem.Pdf/IO/Filters/Ascii85Filter.cs @@ -18,8 +18,10 @@ internal static class Ascii85Filter /// What the data holds that the format does not allow. /// The most the data may decode to. /// - /// Whether the reader cut the data short at one of its limits: a group left partial at its end, before any - /// ~, is one the cut parted, and is dropped rather than completed. + /// Whether the data ends where the file's may not — the reader cut it short at one of its limits, or a guard hides + /// where it ends —: a group left partial at its end, before any ~, is one the cut parted, and is dropped rather + /// than completed, though the bytes it encodes count against , which + /// then says. /// public static byte[] Decode( ReadOnlySpan data, out bool limited, out DataFaults faults, int maxLength, bool cut = false) @@ -142,6 +144,12 @@ public static byte[] Decode( limited = !WriteGroup(output, (uint)tuple, count - 1, group); } + else if (count > 1) + { + // Data that ends where the file's may not leaves a group it parts, whose bytes the missing digits decide: none is + // written. The whole data holds at least the count-1 bytes it encodes, which the bound counts all the same. + limited = count - 1 > maxLength - output.Count; + } if (!ended) { diff --git a/src/AdCodicem.Pdf/IO/Filters/AsciiHexFilter.cs b/src/AdCodicem.Pdf/IO/Filters/AsciiHexFilter.cs index 84b126f6..3f25545e 100644 --- a/src/AdCodicem.Pdf/IO/Filters/AsciiHexFilter.cs +++ b/src/AdCodicem.Pdf/IO/Filters/AsciiHexFilter.cs @@ -18,8 +18,10 @@ internal static class AsciiHexFilter /// What the data holds that the format does not allow. /// The most the data may decode to. /// - /// Whether the reader cut the data short at one of its limits: a digit left alone at its end, before any - /// >, is the first of a pair the cut parted, and is dropped rather than completed with a zero. + /// Whether the data ends where the file's may not — the reader cut it short at one of its limits, or a guard hides + /// where it ends —: a digit left alone at its end, before any >, is the first of a pair the cut parted, and is + /// dropped rather than completed with a zero, though the byte counts against , which + /// then says. /// public static byte[] Decode( ReadOnlySpan data, out bool limited, out DataFaults faults, int maxLength, bool cut = false) @@ -86,6 +88,12 @@ public static byte[] Decode( limited = true; } } + else if (high >= 0) + { + // Data that ends where the file's may not leaves its last byte half read, which the next digit decides: it is + // not written, and the whole data, which holds it, counts it against the bound all the same. + limited = length >= maxLength; + } return buffer.AsSpan(0, length).ToArray(); } diff --git a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs index ac4f9ece..080e6b54 100644 --- a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs +++ b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs @@ -68,8 +68,11 @@ public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diag /// /// Whether where the data ends is the guard's to hide: data a search took up to an endstream nothing confirmed, /// where the length that would say is one a guard cut, or a guard on decoding kept the data from being taken again to - /// it. Only what decoding says of the end hears of it: the data decodes as data whose end the file gives — not as data - /// a guard cut, whose last group a filter drops —, so that whether a guard is reached does not depend on it. + /// it. The data decodes as data a guard cut does — a last group a filter holds in part, which the hidden end may have + /// parted, is dropped, and a predictor's last row undone as far as it goes —, though it is not counted as cut by it: a + /// group dropped so counts against all the same, the + /// whole data holding at least as much, so that whether the guard is reached does not depend on where the end is + /// hidden. /// /// As the other overload gives it. /// As the other overload gives it. @@ -137,9 +140,10 @@ private static ReadOnlyMemory DecodeChain( // Whether the data a filter is given ends where the file's may not: data the reader cut at one of its limits — // MaxStreamLength, or the window a guard stopped at, cut the stream's data, or a filter before it stopped at - // MaxDecodedStreamLength. Every filter after it is given such data, and a cut is reported once, by the guard that - // made it. Data whose end a guard hides is no cut: every filter is given it, and says nothing of its end. - var cut = stream.Data.CutByGuard; + // MaxDecodedStreamLength —, or data whose end a guard hides. Every filter after it is given such data, and a cut is + // reported once, by the guard that made it; data whose end is hidden is no cut. A filter drops a last group such + // data holds in part, and counts it against the bound all the same (Ascii85Filter, AsciiHexFilter). + var cut = stream.Data.CutByGuard || endHidden; var limitReported = false; // The filters before the last share one bound for what they pass along together, and the last keeps its own: a @@ -167,7 +171,6 @@ private static ReadOnlyMemory DecodeChain( diagnostics, stream.Data.Position, guard, - endHidden, ref cut, ref limitReported, out var failed); @@ -206,10 +209,6 @@ public static bool IsImageFilter(PdfName name) => /// Receives what decoding met. /// Where the stream's data starts, where every report is placed. /// Says what reaching the bound does. - /// - /// Whether where ends is the guard's to hide: the filter decodes it as it is, and says nothing of - /// its end. - /// /// /// Whether is data the reader cut; set when this filter stops at the bound too. /// @@ -222,7 +221,6 @@ private static ReadOnlyMemory ApplyOne( PdfDiagnostics? diagnostics, long position, PdfLimitGuard guard, - bool endHidden, ref bool cut, ref bool limitReported, out bool failed) @@ -230,10 +228,6 @@ private static ReadOnlyMemory ApplyOne( var name = step.Name; var maxLength = bounds.MaxLength; var given = cut; - - // Whether how the data ends is the reader's, or the guard's, rather than the file's: data the reader cut, or whose - // end a guard hides. Only what is reported depends on the latter. - var endUnknown = given || endHidden; failed = false; if (name == PdfName.FlateDecode) @@ -251,18 +245,18 @@ private static ReadOnlyMemory ApplyOne( } // Data a guard of the reader's cut ends early because the reader stopped reading it, or a filter before - // this one stopped decoding it, which the guard reported: that it ran out says nothing of the file. A fault, - // or a checksum that disagrees, lies in bytes the reader did read, and the whole data would meet it at the - // same point: it is the file's. + // this one stopped decoding it, which the guard reported, and data whose end a guard hides ends where the guard + // may: that it ran out says nothing of the file. A fault, or a checksum that disagrees, lies in bytes the reader + // did read, and the whole data would meet it at the same point: it is the file's. ReportEnding( - endUnknown && ending is FlateEnding.TailLost or FlateEnding.ChecksumMissing ? FlateEnding.Whole : ending, + given && ending is FlateEnding.TailLost or FlateEnding.ChecksumMissing ? FlateEnding.Whole : ending, decoded.Length, faultAt, data.Length, diagnostics, position); Reached(limited, name, bounds.Shared, diagnostics, position, guard, ref cut, ref limitReported); - return ApplyPredictor(decoded, step, diagnostics, position, cut, endHidden); + return ApplyPredictor(decoded, step, diagnostics, position, cut); } if (name == PdfName.LZWDecode) @@ -279,16 +273,16 @@ private static ReadOnlyMemory ApplyOne( $"An LZW stream uses code {undefinedCode}, which it has not defined; what decoded before it was kept."), position); } - else if (ending == DataEnding.Unmarked && !endUnknown) + else if (ending == DataEnding.Unmarked && !given) { - // Data the reader cut ends early because the reader stopped reading it: that no code ends it says nothing - // of the file. + // Data the reader cut ends early because the reader stopped reading it, and data whose end a guard hides + // ends where the guard may: that no code ends it says nothing of the file. diagnostics?.Repair( PdfDiagnosticCodes.FilterFailed, "An LZW stream ends without its end-of-data code; it decoded to the end of its data.", position); } - else if (ending == DataEnding.UnmarkedInsideCode && !endUnknown) + else if (ending == DataEnding.UnmarkedInsideCode && !given) { diagnostics?.Warn( PdfDiagnosticCodes.FilterFailed, @@ -297,7 +291,7 @@ private static ReadOnlyMemory ApplyOne( } Reached(limited, name, bounds.Shared, diagnostics, position, guard, ref cut, ref limitReported); - return ApplyPredictor(decoded, step, diagnostics, position, cut, endHidden); + return ApplyPredictor(decoded, step, diagnostics, position, cut); } byte[] text; @@ -318,9 +312,9 @@ private static ReadOnlyMemory ApplyOne( text = RunLengthFilter.Decode(data.Span, out textLimited, out faults, maxLength); } - // The end of data the reader cut, or of data the filter stopped decoding at the bound, is the reader's: only what - // the filter read is the file's. - ReportFaults(faults, name, data.Length, textLimited, ended: !endUnknown && !textLimited, diagnostics, position); + // The end of data the reader cut, or whose end a guard hides, or of data the filter stopped decoding at the bound, is + // the reader's: only what the filter read is the file's. + ReportFaults(faults, name, data.Length, textLimited, ended: !given && !textLimited, diagnostics, position); Reached(textLimited, name, bounds.Shared, diagnostics, position, guard, ref cut, ref limitReported); return text; } @@ -336,7 +330,8 @@ private static ReadOnlyMemory ApplyOne( /// Whether the filter stopped at the bound, the faults it counts being those of the part it read. /// /// Whether the filter read its data to the end the file gave it, which its end then says something of: an end marker - /// missing, or a ~ no > follows, where the reader cut the data or the filter stopped is the cut's. + /// missing, or a ~ no > follows, where the reader cut the data, a guard hides its end, or the filter + /// stopped is the cut's. /// /// Receives the reports. /// Where the stream's data starts. @@ -563,12 +558,8 @@ private static void Reached( /// Whether the bound the filters before the last share holds it to less than the bound itself. private readonly record struct Bounds(int MaxLength, bool Shared); - /// - /// Undoes the predictor names, if any: data shorter than one of its rows, whose end the file - /// gives, is left as decoded, and reported unless a guard hides where it ends (). - /// private static ReadOnlyMemory ApplyPredictor( - byte[] decoded, in FilterStep step, PdfDiagnostics? diagnostics, long position, bool cut, bool endHidden) + byte[] decoded, in FilterStep step, PdfDiagnostics? diagnostics, long position, bool cut) { var parameters = step.Parameters; @@ -578,8 +569,7 @@ private static ReadOnlyMemory ApplyPredictor( } if (!PredictorTransform.TryApply( - decoded, parameters.Predictor, parameters.Colors, parameters.BitsPerComponent, parameters.Columns, cut, out var result) && - !endHidden) + decoded, parameters.Predictor, parameters.Colors, parameters.BitsPerComponent, parameters.Columns, cut, out var result)) { diagnostics?.Warn( PdfDiagnosticCodes.FilterFailed, diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 9e749a42..f513c99c 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -639,6 +639,111 @@ byte[] Data(long four, long stream) return file.ToArray(); } + /// + /// A file of one cross-reference stream, object 7, in ASCII85, whose /Length names object 5, which only the + /// stream's own row places, holding the length after 1,015 spaces: a MaxObjectLength of 1,024 parts it after its + /// first digit. Its rows are those of objects 0, 5, 1 to 3, 20 and 21, of reserved types, and 7 (/Index). The text + /// spells an end-of-line and endstream four digits into the group that holds the page's row's last three bytes and + /// row 20's type, which is chosen so that the group's fifth digit is the e of endstream: one that a + /// u in its place carries into the page's row's generation where , and that it does not + /// otherwise. The file is valid. + /// + public static byte[] Ascii85RowsPartedBySpelledEndStream(bool carry) + { + for (var pad = 0; ; pad++) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Text("%" + new string('-', pad) + "\n"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + var low = (int)(file.OffsetOf(3) & 0xFF); + + // The group [low 00 00 type] sums to low + type modulo 85, 2^24 being 1 there: its fifth digit is "e", 68, for one + // type in each range. A "u" in its place adds 16 to the group, which carries into the generation past 239. + var type = -1; + + for (var candidate = carry ? 240 : 200; candidate <= (carry ? 255 : 239) && type < 0; candidate++) + { + type = (low + candidate) % 85 == 68 ? candidate : -1; + } + + if (type < 0) + { + continue; + } + + // The digits "ndstr" and "eam!!" that follow the group are rows 20 and 21's bytes. + string Text(long five, long stream) => + Ascii85(Encode([(0, 0, 65535), (1, five, 0), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0)], Rows.Raw)) + + Ascii85([1, (byte)(file.OffsetOf(3) >> 24), (byte)(file.OffsetOf(3) >> 16), (byte)(file.OffsetOf(3) >> 8)]) + + Ascii85([(byte)low, 0, 0, (byte)type])[..4] + "\nendstr" + "eam\n!!" + Ascii85([.. new byte[5], .. Encode([(1, stream, 0)], Rows.Raw)]) + "~>"; + + file.Object(5, new string(' ', 1015) + Text(0, 0).Length.ToString(CultureInfo.InvariantCulture)); + var at = file.Position; + file.Stream( + 7, + "/Type /XRef /Size 22 /W [1 4 2] /Index [0 1 5 1 1 3 20 2 7 1] /Filter /ASCII85Decode /Root 1 0 R /Length 5 0 R", + Encoding.ASCII.GetBytes(Text(file.OffsetOf(5), at))); + file.StartXRef(at); + return file.ToArray(); + } + } + + /// + /// A file of two revisions. The first holds the catalog, the page tree, the page, object 5, which holds the length of + /// cross-reference stream 7 after 1,015 spaces — a MaxObjectLength of 1,024 parts it after its first digit —, and + /// stream 7, Flate in a stored block with a PNG predictor of columns, whose rows are the + /// catalog's, two of reserved types that spell endstream after an end-of-line, then the page tree's, the page's, + /// object 5's and its own (/Index [1 1 20 2 2 2 5 1 7 1]). The second is a classic table that places object 5 + /// and names stream 7 through /Prev. The file is valid. + /// + public static byte[] PredictedRowsPartedBySpelledEndStream(int columns) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + + // The data's length does not depend on the offsets its rows hold. + byte[] Data(long five, long stream) + { + var rows = new List(Encode([(1, file.OffsetOf(1), 0)], Rows.Raw)); + rows.AddRange("\nendstream\0\0\0\0"u8.ToArray()); + rows.AddRange(Encode([(1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), (1, five, 0), (1, stream, 0)], Rows.Raw)); + + while (rows.Count % columns != 0) + { + rows.Add(0); + } + + var predicted = new List(); + + for (var i = 0; i < rows.Count; i += columns) + { + predicted.Add(0); + predicted.AddRange(rows.GetRange(i, columns)); + } + + return Compress([.. predicted]); + } + + file.Object(5, new string(' ', 1015) + Data(0, 0).Length.ToString(CultureInfo.InvariantCulture)); + var at = file.Position; + file.Stream( + 7, + string.Create( + CultureInfo.InvariantCulture, + $"/Type /XRef /Size 22 /W [1 4 2] /Index [1 1 20 2 2 2 5 1 7 1] /Filter /FlateDecode /DecodeParms << /Predictor 12 /Columns {columns} >> /Root 1 0 R /Length 5 0 R"), + Data(file.OffsetOf(5), at)); + file.Text("%" + new string('-', 1200) + "\n"); + var xref = file.Position; + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"xref\n0 1\n0000000000 65535 f\r\n5 1\n{file.OffsetOf(5):D10} 00000 n\r\ntrailer\n<< /Size 22 /Root 1 0 R /Prev {at} >>\nstartxref\n{xref}\n%%EOF\n")); + return file.ToArray(); + } + /// /// A file of one cross-reference stream, object 7, whose /Length names object 5, which only the stream's own row /// places, before the rows of objects 1 and 2; two rows of reserved types spell endstream after an end-of-line @@ -942,6 +1047,34 @@ private static byte[] Encode((int Type, long Second, int Third)[] rows, Rows enc return encoding == Rows.Raw ? [.. data] : Compress([.. data]); } + /// Writes in ASCII85, whole groups but the last, with no z and no ~>. + private static string Ascii85(ReadOnlySpan bytes) + { + var text = new StringBuilder(); + Span digits = stackalloc char[5]; + + for (var i = 0; i < bytes.Length; i += 4) + { + var count = Math.Min(4, bytes.Length - i); + uint value = 0; + + for (var j = 0; j < 4; j++) + { + value = (value << 8) | (j < count ? bytes[i + j] : 0u); + } + + for (var j = 4; j >= 0; j--) + { + digits[j] = (char)('!' + (value % 85)); + value /= 85; + } + + text.Append(digits[..(count + 1)]); + } + + return text.ToString(); + } + /// Compresses data in stored blocks, which show it as written. private static byte[] Compress(byte[] data) { diff --git a/tests/AdCodicem.Pdf.Tests/FilterTests.cs b/tests/AdCodicem.Pdf.Tests/FilterTests.cs index f9fccca2..1b6415ef 100644 --- a/tests/AdCodicem.Pdf.Tests/FilterTests.cs +++ b/tests/AdCodicem.Pdf.Tests/FilterTests.cs @@ -26,6 +26,35 @@ public class FilterTests 0x49, 0x2D, 0x2A, 0x56, 0xF4, 0x20, 0x4A, 0x08, 0x00, ]; + [Theory] + [InlineData(4, true)] + [InlineData(6, false)] + public void A_group_ascii85_data_that_ends_where_the_file_s_may_not_parts_is_dropped_and_counted_against_the_bound(int maxLength, bool limited) + { + // "5sdq," is ABCD, and "77I" the part of the group EF's: data the reader cut, or whose end a guard hides, may end + // inside the group, whose bytes the missing digits decide. They are not written, but the whole data holds them, + // and a bound they do not fit within is reached (#224). + var decoded = Ascii85Filter.Decode("5sdq,77I"u8, out var reached, out _, maxLength, cut: true); + + decoded.Should().Equal("ABCD"u8.ToArray()); + reached.Should().Be(limited); + Ascii85Filter.Decode("5sdq,77I"u8, out _, out _, 6).Should().Equal("ABCDEF"u8.ToArray()); + } + + [Theory] + [InlineData(2, true)] + [InlineData(3, false)] + public void A_digit_asciihex_data_that_ends_where_the_file_s_may_not_leaves_alone_is_dropped_and_counted_against_the_bound(int maxLength, bool limited) + { + // The last digit's byte, which the next digit decides, is not written, but the whole data holds it, and a bound it + // does not fit within is reached (#224). + var decoded = AsciiHexFilter.Decode("41424"u8, out var reached, out _, maxLength, cut: true); + + decoded.Should().Equal("AB"u8.ToArray()); + reached.Should().Be(limited); + AsciiHexFilter.Decode("41424"u8, out _, out _, 3).Should().Equal([0x41, 0x42, 0x40]); + } + [Fact] public void Decodes_a_flate_stream() { diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 278337bb..35502891 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -976,6 +976,56 @@ public void An_R_a_window_s_edge_reaches_makes_a_reference_only_where_the_file_e document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard(bool carry) + { + // Stream 7's /Length, which only its own row places, is one MaxObjectLength parts after its first digit, and its + // ASCII85 text spells an endstream four digits into the group that holds the page's row's last bytes: nothing after + // it proves it the stream's own, and where the data ends is the guard's. Decoded as data whose end the file gives, + // the group was completed with u's, which guessed its bytes — and carried into the page's row's generation, read + // then as 1 —: the row the parted group holds is the guard's, the page read as null until the guard is lifted (#224). + var file = ChainFiles.Ascii85RowsPartedBySpelledEndStream(carry); + + using (var whole = PdfDocument.Open(file)) + { + whole.Diagnostics.Should().BeEmpty(); + whole.GetObject(new PdfObjectId(3)).AsDictionary().Required().IsOfType(PdfName.Page).Should().BeTrue(); + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + document.GetObject(new PdfObjectId(3)).Should().BeOfType(); + document.Reader.Structure.Sections.Should().ContainSingle().Which.CutByLimit.Should().BeTrue(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + new PdfValidator().Validate(document).Findings.Select(finding => finding.RuleId).Should().NotContain(PdfValidationRuleIds.XRefGenerationMismatch); + } + + [Theory] + [InlineData(20)] + [InlineData(49)] + public void A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes(int columns) + { + // Stream 7's /Length, object 5, which the newer table places, is one MaxObjectLength parts after its first digit, and + // its rows, Flate behind a PNG predictor, spell an endstream after the catalog's: the data up to it decodes to the + // catalog's row behind its tag, shorter than one predicted row. Decoded as data whose end the file gives, it was left + // as decoded, the tag read as the catalog's row's type, the catalog freed and the index rebuilt. Where the end is + // the guard's, the row is undone as far as the data goes, and the catalog read (#224). + var file = ChainFiles.PredictedRowsPartedBySpelledEndStream(columns); + + using (var whole = PdfDocument.Open(file)) + { + whole.Diagnostics.Should().BeEmpty(); + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.GetObject(new PdfObjectId(1)).AsDictionary().Required().IsOfType(PdfName.Catalog).Should().BeTrue(); + } + [Theory] [InlineData(29)] [InlineData(20)] From bbf30e742fed8a4030c49531124b80aadb3e55e4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 10:06:29 +0000 Subject: [PATCH 089/146] fix: check a deferred length against data that holds every row whatever a guard on decoding did The chain left a cross-reference stream's deferred /Length unchecked wherever a guard on decoding stopped the data, on the ground that the guard kept the data from being taken again. Data that holds every row the section declares is taken again by nothing: its filter's run-out at an endstream the data spells stood as the file's fault, and nothing said the chain took the data short, as the whole read does. The length is now checked there too, but not a length a guard cut. Refs #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 9 ++- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 61 +++++++++++++++++++ .../DocumentReaderTests.cs | 17 ++++++ 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index c0ab1942..8c360e28 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3976,9 +3976,12 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect } } - // Data a guard cut — MaxStreamLength as the data was read, or a guard on decoding it — is the guard's, which says so - // (ADR 34), and is not checked against a length: the cut kept the data from being taken again to it. - if (deferred is { } check && decodedWhole) + // Data a guard cut — MaxStreamLength as the data was read, or a guard on decoding it that left rows lacking — is the + // guard's, which says so (ADR 34), and is not checked against a length: the cut kept the data from being taken again + // to it. Data that holds every row the section declares, which nothing would take again, is checked whatever a guard + // on decoding did, but not against a length a guard cut, which is no length the file gave. + if (deferred is { } check && + (decodedWhole || (!xrefStream.Data.CutByGuard && !lengthCut && declared <= held))) { (_deferredLengths ??= [])[xrefStream.Data.Position] = check; } diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index f513c99c..75a254c4 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -639,6 +639,67 @@ byte[] Data(long four, long stream) return file.ToArray(); } + /// + /// A file of two revisions, as writes it, but stream 5 is + /// /Filter [/RunLengthDecode /FlateDecode]: a first literal run of RunLength holds the Flate data, which decodes to + /// its rows and 1,000 zero bytes, and a second the bytes of an end-of-line, endstream, another end-of-line and + /// XYZ, where a deferred search stops. Its /Length names object 4, which only the stream's own row places, + /// unless is false: written direct. The file is valid. + /// + public static byte[] UpdateOverRunsSpellingAnEndStream(bool deferred) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + + // Compressed, the data's length depends on the offsets its rows hold: written in four digits, it does not move them. + byte[] Data(long four, long stream) + { + var rows = Encode( + [(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), (1, four, 0), (1, stream, 0)], + Rows.Raw); + using var output = new MemoryStream(); + + using (var zlib = new ZLibStream(output, CompressionLevel.Optimal, leaveOpen: true)) + { + zlib.Write([.. rows, .. new byte[1000]]); + } + + var flate = output.ToArray(); + var data = new List(); + + for (var i = 0; i < flate.Length; i += 128) + { + var count = Math.Min(128, flate.Length - i); + data.Add((byte)(count - 1)); + data.AddRange(flate.AsSpan(i, count).ToArray()); + } + + var spelled = "\nendstream\nXYZ"u8.ToArray(); + data.Add((byte)(spelled.Length - 1)); + data.AddRange(spelled); + data.Add(128); + return [.. data]; + } + + var four = file.Position; + var at = four + "4 0 obj\n0000\nendobj\n".Length; + var data = Data(four, at); + var length = data.Length.ToString("D4", CultureInfo.InvariantCulture); + file.Object(4, length); + file.Stream( + 5, + $"/Type /XRef /Size 6 /W [1 4 2] /Filter [/RunLengthDecode /FlateDecode] /Root 1 0 R /Length {(deferred ? "4 0 R" : length)}", + data); + file.Text("%" + new string('-', 1200) + "\n"); + var xref = file.Position; + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"xref\n0 4\n0000000000 65535 f\r\n{file.OffsetOf(1):D10} 00000 n\r\n{file.OffsetOf(2):D10} 00000 n\r\n{file.OffsetOf(3):D10} 00000 n\r\ntrailer\n<< /Size 6 /Root 1 0 R /Prev {at} >>\nstartxref\n{xref}\n%%EOF\n")); + return file.ToArray(); + } + /// /// A file of one cross-reference stream, object 7, in ASCII85, whose /Length names object 5, which only the /// stream's own row places, holding the length after 1,015 spaces: a MaxObjectLength of 1,024 parts it after its diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 5dac28a1..adc5173d 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -2005,6 +2005,23 @@ public void A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedSt deferred.Reader.Structure.Sections[^1].Incomplete.Should().BeFalse(); } + [Fact] + public void A_deferred_length_s_data_that_holds_every_row_under_a_guard_on_decoding_is_checked_against_its_length() + { + // Stream 5's RunLength runs hold its Flate rows and zero bytes, then the bytes of a spelled endstream, where the + // deferred search stops; MaxDecodedStreamLength stops the Flate output past every row. Nothing would take the data + // again, and the RunLength run the spelled endstream parts is reported as read whole, but the length, which says the + // chain took the data short, was not checked under the guard: the run-out stood as the file's fault alone (#220). + var file = ChainFiles.UpdateOverRunsSpellingAnEndStream(deferred: true); + using var whole = PdfDocument.Open(file); + using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxDecodedStreamLength = 100 } }); + + whole.Diagnostics.Should().Contain(entry => entry.Code == PdfDiagnosticCodes.StreamLengthInvalid); + document.Diagnostics.Select(entry => entry.Code).Where(code => code != PdfDiagnosticCodes.LimitDecodedStream) + .Should().Equal(whole.Diagnostics.Select(entry => entry.Code)); + document.Diagnostics.Should().Contain(entry => entry.Code == PdfDiagnosticCodes.LimitDecodedStream); + } + [Fact] public void A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends() { From 154464460781504fdb227c464f0a1e84a75b4293 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 10:19:13 +0000 Subject: [PATCH 090/146] fix: prove an endstream a stream's own against each reader's index, and nothing past the budget The proof R6-1 asks of an endstream under a hidden length had five gaps: - An endstream past which the searches could read no more (EndObj Unseen) skipped the proof: its data, cut short at a spelled endstream, was the file's. - The answer was kept per data end, though it hangs on the caller's next object: a first caller that knew of none fixed it for every later load. - The chain proved with the sections read before it alone, though the rows the stream holds place the object that follows it. - "endstreamxref", one token and no endstream, proved itself through the keyword glued to it. - A row naming the white space before the next object's header, which the reader reads the object at all the same, proved nothing. Now nothing past the budget proves anything, and what follows the object's end is kept, with whether a section keyword or the end of the file stands there. Each caller's next object is weighed against that position, a row on the white space before a header included. The chain proves after the data taken again is decoded, the rows it holds placing objects too. A whole endstream keyword is required. The proof's reads start at 16 bytes, which the search budget counts. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 122 +++++++++++++----- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 7 +- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 41 ++++++ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 70 +++++++++- .../AdCodicem.Pdf.Tests/StreamLengthTests.cs | 31 +++++ 5 files changed, 232 insertions(+), 39 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 8c360e28..889c0093 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -150,7 +150,10 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// searched once to its endstream, or to a header its data spells, which leaves the data up to it to the guard; /// and white space after a declared length that runs past a first read is read on once for each place (#174). The data /// of different streams lies apart, and so do the runs of white space after them: together those reads cover the file - /// once, and one read past each — about twice the file at most. Every other search is of an invalid stream, whose + /// once, and one read past each — about twice the file at most. A fourth adds a little to each stream of the second + /// way: the proof that an endstream found is the stream's own (#224) reads past it to its object's end and what + /// follows, three first reads of bytes, which double only as white space runs on — a few times + /// a stream of a few bytes, so that a file of such streams alone reads about three times itself. Every other search is of an invalid stream, whose /// endstream does not follow its declared length. Each /// reads from a stream's data to the first object header the bytes hold after it, or to the next object the index as /// written places, whichever is nearer, and its reads grow from twice what the parser holds of the data @@ -165,6 +168,12 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// internal const int EndStreamSearchPasses = 4; + /// + /// How many bytes the first read of each step of an endstream's proof reads (): + /// what follows a keyword is mostly an end-of-line away, and the reads double past it as white space runs on. + /// + private const int ProofRead = 16; + /// Asks for a direct object, with no object header: a trailer. private const int DirectObject = -1; @@ -389,10 +398,12 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider private readonly Dictionary _endStreamChecks = []; /// - /// Whether the endstream after each place a search ended data at is the stream's own beyond what its length - /// could say (): read once, however many times its stream is parsed. + /// What follows the object's end after each place a search ended data at, with what was said follows the + /// endstream there, as reads it: where its first significant + /// byte lies, -1 where nothing could be read, and whether the end of the file or a cross-reference section's keyword + /// stands there. Read once, however many times its stream is parsed; each caller's next object is weighed against it. /// - private readonly Dictionary _ownEndStreams = []; + private readonly Dictionary<(long DataEnd, EndObjState EndObj), (long Position, bool Boundary)> _endStreamFollowers = []; /// /// Where the data starts of each stream whose length fault a reading the reader kept reported: the stream is not @@ -1724,11 +1735,12 @@ private bool IsEndObjAt(long absoluteOffset) /// Gives where the first byte at or after lies that is neither white space nor part of a /// comment, reading the file on as far as they run, or the length of the file when they run to its end; -1 when the /// searches for endstream have read as much as they may (), which these reads - /// count with, or the source gives less than it holds. + /// count with, or the source gives less than it holds. The first read is bytes, and each + /// after it twice the one before, up to . /// - private long NextSignificant(long position) + private long NextSignificant(long position, int firstRead = PdfObjectParser.EndObjLookahead) { - var size = PdfObjectParser.EndObjLookahead; + var size = firstRead; var inComment = false; while (position < _source.Length) @@ -1896,28 +1908,49 @@ private StreamEndSearch SearchEndStreamOnce(int number, long absoluteDataStart, /// bool IPdfStreamDataProvider.IsOwnEndStream(long absoluteDataEnd, EndObjState endObj, long? nextObject) { - if (_ownEndStreams.TryGetValue(absoluteDataEnd, out var known)) + var (position, boundary) = EndStreamFollower(absoluteDataEnd, endObj); + return position >= 0 && (boundary || IsPlacedAt(position, nextObject)); + } + + /// + /// Reads, once for each place, what follows the end of the object whose data a search ended at + /// : past the endstream keyword, whole, and the endobj after it where + /// says one follows, white space and comments between, through first reads of + /// bytes counted with the searches (). + /// + private (long Position, bool Boundary) EndStreamFollower(long dataEnd, EndObjState endObj) + { + if (_endStreamFollowers.TryGetValue((dataEnd, endObj), out var known)) { return known; } - // The endstream keyword, then the endobj that follows it, where one does: each found where a reading said it lies. - var end = NextSignificant(absoluteDataEnd); - end = end < 0 || end >= _source.Length ? -1 : end + StreamEnds.EndStreamKeyword.Length; + // "endstreamxref" is no keyword, and ends no data: what follows it proves nothing. + var keyword = NextSignificant(dataEnd, ProofRead); + var end = keyword >= 0 && keyword < _source.Length && IsKeywordAt(keyword, StreamEnds.EndStreamKeyword) + ? keyword + StreamEnds.EndStreamKeyword.Length + : -1; if (end >= 0 && endObj == EndObjState.Present) { - end = NextSignificant(end); + end = NextSignificant(end, ProofRead); end = end < 0 || end >= _source.Length ? -1 : end + StreamEnds.EndObjKeyword.Length; } - var next = end < 0 ? -1 : NextSignificant(end); - var own = next >= 0 && - (next == _source.Length || next == nextObject || IsKeywordAt(next, "xref"u8) || IsKeywordAt(next, StartXRefKeyword)); - _ownEndStreams[absoluteDataEnd] = own; - return own; + var next = end < 0 ? -1 : NextSignificant(end, ProofRead); + var follower = (next, next >= 0 && + (next == _source.Length || IsKeywordAt(next, "xref"u8) || IsKeywordAt(next, StartXRefKeyword))); + _endStreamFollowers[(dataEnd, endObj)] = follower; + return follower; } + /// + /// Determines whether the object an index places at starts at : + /// there, or past the white space and comments its row names, as the reader reads an object at a row's offset. + /// + private bool IsPlacedAt(long position, long? placed) => + placed is { } at && (at == position || (at < position && NextSignificant(at, ProofRead) == position)); + /// /// Determines whether , whole, starts at : the file ends after it, /// or no regular character follows. @@ -2184,8 +2217,8 @@ internal void Close() _endStreamSearches.TrimExcess(); _endStreamChecks.Clear(); _endStreamChecks.TrimExcess(); - _ownEndStreams.Clear(); - _ownEndStreams.TrimExcess(); + _endStreamFollowers.Clear(); + _endStreamFollowers.TrimExcess(); Release(_lengthFaultsReported); Release(_syntaxReported); Release(_limitsReached); @@ -3892,23 +3925,24 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // is asked too: a newer section may place the object, which a guard cut. if (!retaken && !endHidden && deferred is { } unsettled && declared > held && decodedWhole) { - var whole = RetakeDeferred(xrefStream, unsettled, data, fieldWidths, row, held, index is null ? [0, declared] : subsections, out lengthCut); - - // Where a guard cut the length, the endstream the data ends at is the stream's own, and the rows it lacks - // the file's, only where what follows proves it, as a load of the stream reads it (#224). - lengthCut = lengthCut && !EndsAtOwnEndStream(whole ?? xrefStream); - - if (whole is not null) + if (RetakeDeferred(xrefStream, unsettled, data, fieldWidths, row, held, index is null ? [0, declared] : subsections, out lengthCut) is { } whole) { _diagnostics.RollBack(decoding); xrefStream = whole; deferred = unsettled with { Taken = whole.Data.Length }; retaken = true; - endHidden = lengthCut; continue; } } + // Where a guard cut the length, the endstream the data ends at — as the chain took it, or took it again — is the + // stream's own, and the rows it lacks the file's, only where what follows proves it, as a load of the stream + // reads it: the rows the data holds place objects too (#224). + if (lengthCut && !endHidden && EndsAtOwnEndStream(xrefStream, data, fieldWidths, row, held)) + { + lengthCut = false; + } + // Data a /Length the chain could not read left to an endstream nothing confirmed, where the length that would say // whether it is the data's own is one a guard cut, or where a guard on decoding kept the data, which lacks rows, // from being taken again to the length: where it ends is the guard's, and a filter run out there says nothing of @@ -4008,17 +4042,41 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect /// /// Determines whether the endstream cross-reference stream 's data ends at, as the chain /// took it or took it again, is the stream's own beyond what a /Length a guard cut could say - /// (): where the data ends is then the file's (#224). + /// (): where the data ends is then the file's (#224). The next + /// object an index places is the nearer of the one the sections read so far place and the first the + /// rows of place past the start of the data, as the index a load of the + /// stream reads with holds them. /// - private bool EndsAtOwnEndStream(PdfStream stream) + private bool EndsAtOwnEndStream(PdfStream stream, ReadOnlySpan data, ReadOnlySpan widths, int row, long held) { var dataStart = stream.Data.Position; var dataEnd = dataStart + stream.Data.Length; var provider = (IPdfStreamDataProvider)this; - return _endStreamSearches.TryGetValue((dataStart, true), out var known) && - provider.CheckEndStream(dataEnd) is EndObjState endObj and (EndObjState.Absent or EndObjState.Present) && - provider.IsOwnEndStream(dataEnd, endObj, known.Search.NextObject); + if (!_endStreamSearches.TryGetValue((dataStart, true), out var known) || + provider.CheckEndStream(dataEnd) is not EndObjState endObj || endObj is not (EndObjState.Absent or EndObjState.Present)) + { + return false; + } + + var next = known.Search.NextObject; + + // Each row lies within the data, whose length an int holds. + for (var at = 0L; at < held; at++) + { + var fields = data.Slice((int)at * row, row); + var cursor = 0; + var type = widths[0] == 0 ? 1 : ReadField(fields, ref cursor, widths[0]); + var offset = ReadField(fields, ref cursor, widths[1]); + + if (type == 1 && offset <= long.MaxValue && PositionOf((long)offset) is var placed && placed > dataStart && + (next is null || placed < next)) + { + next = placed; + } + } + + return provider.IsOwnEndStream(dataEnd, endObj, next); } /// diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 337cd511..dc1112d6 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -838,9 +838,10 @@ private PdfStream ReadStream(PdfDictionary dictionary) // apart is the guard's. One what follows proves its own — the next object the index places, the end of the file, // a cross-reference section's keyword, after the object's endobj or in its place — ends the data, the file's. // Any other's data is the guard's, cut by it, the object's endobj unjudged, and nothing is said of the file, as - // at a header the data spells (#224). - if (hidden && settled.EndObj is EndObjState.Absent or EndObjState.Present && _streamData is { } file && - !file.IsOwnEndStream(_baseOffset + dataStart + found, settled.EndObj, settled.NextObject)) + // at a header the data spells (#224): one past which the searches could read nothing more, too. + if (hidden && _streamData is { } file && + (settled.EndObj is not (EndObjState.Absent or EndObjState.Present) || + !file.IsOwnEndStream(_baseOffset + dataStart + found, settled.EndObj, settled.NextObject))) { _endObjUnknown = true; return Finish(dictionary, dataStart, found, span.Length, cutByGuard: true); diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 75a254c4..3abda3e9 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -700,6 +700,47 @@ byte[] Data(long four, long stream) return file.ToArray(); } + /// + /// A file of one cross-reference stream, object 5, whose /Length names object 4, which only the stream's own row + /// places and which holds the length after 1,100 spaces — a MaxObjectLength of 1,024 cuts it before its value. + /// Its rows, of objects 0 to 2, 4, 5 and 8, then 3 (/Index [0 3 4 2 8 1 3 1]), lose their last 3 bytes: the page's + /// row lacks, the file's fault. With , object 8, which its row places, follows the stream, + /// and the startxref otherwise. + /// + public static byte[] AloneWithACutLengthLackingItsLastRow(bool objectAfter) + { + // Every row and offset is as wide whatever it holds: where object 8 lies is read once, then written. + _ = Build(0, out var eight); + return Build(eight, out _); + + byte[] Build(long eightAt, out long eight) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + var four = file.Position; + var at = four + ("4 0 obj\n" + new string(' ', 1100) + "46\nendobj\n").Length; + var rows = Encode( + [ + (0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, four, 0), (1, at, 0), (1, eightAt, 0), + (1, file.OffsetOf(3), 0), + ], + Rows.Raw); + file.Object(4, new string(' ', 1100) + "46"); + file.Stream(5, "/Type /XRef /Size 9 /W [1 4 2] /Index [0 3 4 2 8 1 3 1] /Root 1 0 R /Length 4 0 R", rows[..^3]); + eight = file.Position; + + if (objectAfter) + { + file.Object(8, "(eight)"); + } + + file.StartXRef(at); + return file.ToArray(); + } + } + /// /// A file of one cross-reference stream, object 7, in ASCII85, whose /Length names object 5, which only the /// stream's own row places, holding the length after 1,015 spaces: a MaxObjectLength of 1,024 parts it after its diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 35502891..b8dce85d 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -474,8 +474,8 @@ public void A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_gua /// A page whose content stream, object 4, holds BT ET, its /Length object 20, which holds 5 after 2,000 /// spaces where is cut; its endstream, and its endobj where /// , followed by : object 6, which the table places, or which no row - /// places; the table; the startxref after the table, which comes before object 4; or the end of the file, past - /// the one the table's %%EOF marks. + /// places — at its header, or on the end-of-line before it —; the table; the startxref after the table, which + /// comes before object 4; or the end of the file, past the one the table's %%EOF marks. /// private static byte[] StreamFollowedBy(string guard, string follower, bool endObj) { @@ -516,9 +516,10 @@ string Text(long at) { var six = file.Object(6, "(six)"); - if (follower == "next object") + // A row may name the white space before the header, where the reader reads the object all the same. + if (follower is "next object" or "next object past white space") { - rows[6] = six; + rows[6] = follower == "next object" ? six : six - 1; } } } @@ -839,6 +840,8 @@ public void An_endstream_the_next_object_follows_where_a_guard_hid_the_length_la [Theory] [InlineData("next object", true)] [InlineData("next object", false)] + [InlineData("next object past white space", true)] + [InlineData("next object past white space", false)] [InlineData("xref", true)] [InlineData("xref", false)] [InlineData("startxref", true)] @@ -976,6 +979,65 @@ public void An_R_a_window_s_edge_reaches_makes_a_reference_only_where_the_file_e document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); } + [Fact] + public void The_proof_of_an_endstream_weighs_each_caller_s_next_object_against_what_follows_its_object() + { + // What follows object 4's end is read once, and each caller's next object weighed against it: the answer kept for a + // first caller that knew of no next object was served to every later one, whose index places object 6 there (#224). + var file = StreamFollowedBy("cut", "next object", endObj: true); + var six = (long)Encoding.Latin1.GetString(file).IndexOf("6 0 obj", StringComparison.Ordinal); + using var document = PdfDocument.Open(file, ObjectGuard); + var data = document.GetObject(new PdfObjectId(4)).AsStream().Required().Data; + var provider = (IPdfStreamDataProvider)document.Reader; + + provider.IsOwnEndStream(data.Position + data.Length, EndObjState.Present, null).Should().BeFalse(); + provider.IsOwnEndStream(data.Position + data.Length, EndObjState.Present, six).Should().BeTrue(); + provider.IsOwnEndStream(data.Position + data.Length, EndObjState.Present, six - 1).Should().BeTrue(); + provider.IsOwnEndStream(data.Position + data.Length, EndObjState.Present, six + 1).Should().BeFalse(); + } + + [Theory] + [InlineData("xref")] + [InlineData("startxref")] + public void An_endstream_the_data_glues_to_a_section_keyword_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s(string keyword) + { + // Object 4's content spells "endstreamxref" in a string, one token and no endstream keyword: the proof stepped over + // nine bytes and found the section's keyword past them, and the data, cut short there, was taken for the file's, its + // object said to lack its endobj. A token that is no endstream proves nothing: the data is the guard's (#224). + var data = "BT (aendstream" + keyword + " ) Tj ET"; + var file = ContentLengthIn(new string(' ', 2000) + data.Length.ToString(CultureInfo.InvariantCulture), data); + + using var document = PdfDocument.Open(file, ObjectGuard); + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + stream.GetRawBytes().ToArray().Should().Equal("BT (a"u8.ToArray()); + stream.Data.CutByGuard.Should().BeTrue(); + document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void Rows_a_cross_reference_stream_whose_length_a_guard_cut_lacks_are_the_file_s_where_what_follows_its_endstream_proves_it(bool objectAfter) + { + // Stream 5's /Length, object 4, which only its own row places, is one MaxObjectLength cuts before its value, and its + // data lacks the page's row. Its endobj is followed by object 8, which only its own rows place, or by the startxref: + // either proves its endstream its own. The chain asked the sections read before it alone, and took the row the data + // lacks for the guard's, the page unread; a load of the stream then kept the chain's answer. The row is the file's + // to lack, as read whole: the index is rebuilt, the page read (#224). + var file = ChainFiles.AloneWithACutLengthLackingItsLastRow(objectAfter); + using var whole = PdfDocument.Open(file); + using var document = PdfDocument.Open(file, ObjectGuard); + + whole.Reader.Structure.Sections[0].Incomplete.Should().BeTrue(); + document.Reader.Structure.Sections[0].Incomplete.Should().BeTrue(); + document.Reader.Structure.Sections[0].CutByLimit.Should().BeFalse(); + document.GetObject(new PdfObjectId(3)).AsDictionary().Required().IsOfType(PdfName.Page).Should().BeTrue(); + document.WasRepaired.Should().BeTrue(); + document.GetObject(new PdfObjectId(5)).AsStream().Required().Data.CutByGuard.Should().BeFalse(); + } + [Theory] [InlineData(false)] [InlineData(true)] diff --git a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs index 0e42a302..e95c8c95 100644 --- a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs +++ b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs @@ -2200,6 +2200,37 @@ private static (byte[] File, string Head) NestedStreams(int streams, int declare return (file, head); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public void An_endstream_the_data_spells_past_which_the_searches_may_read_no_more_ends_data_that_is_the_guard_s(bool endObj) + { + // Ten streams whose data share a megabyte read as much of the file as the searches may. Then stream 13, whose + // /Length, object 14, MaxObjectLength cuts, spells an endstream in a string, a long comment after it, then an endobj + // or not: what follows the comment lies past what the searches may read, and nothing proves the endstream the + // stream's own. The data was taken for the file's, cut short unmarked, its object said to lack its endobj where the + // data spells none; it is the guard's (#224). + const int Streams = 10; + var data = "BT (a\nendstream\n%" + new string('c', 80) + (endObj ? "\nendobj\n" : "\n") + ") Tj ET"; + var (file, _) = NestedStreams( + Streams, + 100_000, + Invariant($"{Streams + 3} 0 obj\n<< /Length {Streams + 4} 0 R >>\nstream\n{data}\nendstream\nendobj\n"), + Invariant($"{Streams + 4} 0 obj\n{new string(' ', 2000)}{data.Length}\nendobj\n")); + using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = 1024 } }); + + for (var number = 3; number <= Streams + 2; number++) + { + document.GetObject(new PdfObjectId(number)); + } + + var stream = document.GetObject(new PdfObjectId(Streams + 3)).AsStream().Required(); + + stream.GetRawBytes().ToArray().Should().Equal("BT (a"u8.ToArray()); + stream.Data.CutByGuard.Should().BeTrue(); + document.Reader.IsEndObjMissing(Streams + 3, out _).Should().BeFalse(); + } + [Fact] public void A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds() { From 798f57e2e6c734f50e5e1f854ccbe72c1591f161 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 10:28:07 +0000 Subject: [PATCH 091/146] fix: take a place a section may have moved to only where a value's guard cut keeps it unread A place near the offset /Prev names was kept as the section whenever a guard cut it. A CIDFont whose /W runs past MaxTrailerLength passed the filter for its /W, reached the trailer guard and was relocated to: the chain ended at a font, the farther table was never tried, and the catalog was lost. Only a value the section is read with, or its /Length, that leads to an object a guard cut now keeps a guessed place, and a guess must declare a cross-reference stream and no other type: a /W beside a /Type that names another is a font's widths. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 2 +- docs/website/docs/reference/reader-limits.md | 3 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 28 +++++++++++++----- .../IO/XRef/XRefSectionRecord.cs | 8 +++++ .../DocumentReaderTests.cs | 29 +++++++++++++++++-- 5 files changed, 59 insertions(+), 11 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index c46abdf4..0e5f48c8 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -282,7 +282,7 @@ Known gaps: | A classic table of any length | Its window grows ×4 from 64 KB up to `MaxXRefSectionLength`, and the rows read are kept. A table the guard cuts, or whose rows run past a full index, keeps its trailer, found past the rows left unread by a search a megabyte at a time (`ReadTrailerPastRows`): comments are skipped, and anything else no row holds is the section's fault, reported, the search going on to the `trailer` keyword, so that its `/Encrypt` still refuses the document. The search starts at the start of the line the cut fell in, outside any comment and past the table's own keyword; it reads each megabyte once, every byte of it looked at before the next is read, however many comments or stray bytes it holds, and is not made for a place a section may have been moved to, a guess. The chain ends there, since an older section's rows would stand for the objects the unread rows define. What those rows place reads as null until the guard is raised | `ReaderLimitsTests.A_table_whose_keywords_the_bound_cuts_reports_the_bound`, `A_table_its_length_bound_cuts_keeps_its_trailer_and_the_encryption_it_names`, `A_table_its_length_bound_cuts_ends_the_chain_though_its_trailer_names_an_older_section`, `What_ends_the_rows_a_length_bound_cut_is_judged_as_it_is_where_none_did`, `A_table_a_length_bound_cuts_that_runs_to_the_end_of_the_file_has_no_trailer`, `A_table_its_length_bound_cuts_keeps_the_encryption_its_trailer_names_past_what_no_row_holds`, `A_table_cut_before_its_keyword_ends_keeps_its_trailer`, `Rows_past_a_full_index_are_not_read_and_the_trailer_past_them_is`, `A_cut_in_a_comment_among_the_rows_searches_for_the_trailer_from_the_comment_s_start`, `The_trailer_past_rows_left_unread_is_found_wherever_the_search_s_window_ends`, `A_trailer_past_rows_left_unread_that_its_own_bound_cuts_keeps_what_precedes_the_cut`; `DocumentReaderTests.The_search_for_the_trailer_past_rows_left_unread_reads_the_file_once`; `CrossReferenceChainTests.Reads_an_object_a_length_bound_cut_out_of_its_table_as_null_until_the_guard_is_lifted`, `A_section_found_near_where_prev_names_it_is_not_searched_past_the_rows_a_guard_left_unread`; `WindowEdgeTests.A_classic_table_reads_whole_wherever_its_window_edge_falls_near_its_trailer` | No: seeds stay under 120 KB | | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). A hybrid file's stream's own `/Prev`, which the chain does not follow, going on through the table's, is not read at all, nor are the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place: neither names a section missing nor ends the chain for a guard, and the rule on trailer values judges them as written, a reference they hold not followed ([#263]). An object the guard cut after a whole reference it holds hides nothing: the chain reads through it, as a load does, and loads it once the chain is read. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s`, `A_table_an_xrefstm_names_in_a_stream_s_place_has_its_own_links_left_unread`, `A_prev_written_as_a_reference_to_a_real_is_judged_through_it_only_where_the_reader_follows_it` | Yes | -| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading, nor what it read through: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps — not for a place it dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]) | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild` | No | +| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading, nor what it read through: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps — not for a place it dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); a place that declares another type — a CIDFont, whose `/W` gives widths —, or whose own dictionary a guard cut, is a guess dropped, the next place tried | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild` | No | | A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, to the same bytes — not as data a guard cut, whose last group ASCII85 and ASCIIHex drop, which could take a decoding guard's cut away and charge the rows to the file —: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index 4f8d8c72..74e16829 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -134,7 +134,8 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. `/Length` does when no `endstream` ends its data, are the guard's, as a section it cut is: the chain ends there, the sections it names are not read — the older ones, or a hybrid file's cross-reference stream —, the index is not rebuilt, and the guard is reported once the chain is read, or thrown by `Open`. So does a section found near the - offset it is named at, which such a value keeps from being read. The rows a cross-reference stream's data lacks, + offset it is named at, which such a value keeps from being read; a place there that declares another type — a CIDFont, + whose `/W` gives widths —, or whose own dictionary a guard cut, is not taken for the section, and the next is tried. The rows a cross-reference stream's data lacks, where its `/Length` so leads and its data was taken to the first `endstream`, are the guard's too, unless what follows that `endstream` proves it the stream's own, as above: they are then the file's to lack, and the index is rebuilt. An object the chain read through a reference the cut left whole is loaded once the chain is read only for a value it diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 889c0093..d7edcf4d 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -2990,8 +2990,10 @@ private void ReportMissingSection(string naming, long offset, long namedFrom) /// places in all, over the whole document: each is read through a window /// of up to 64 KB, the file chooses how many headers lie near a section, and a chain can name a section for /// each of its links. Found, the section's record takes what - /// was read there — a section a guard kept from being read too, which is not found to be read, the chain ending - /// there as the guard's —; not found, it keeps what the named offset held. + /// was read there — a section a value it is read with kept from being read too, leading to an object a guard cut, which + /// is not found to be read, the chain ending there as the guard's —; not found, it keeps what the named offset held. A + /// place is a guess: it must declare a cross-reference stream and no other type, and one a guard cut the dictionary of + /// is dropped. /// private bool TryRelocateXRefSection(long offset, XRefSectionRecord section, out long previous, out long hybrid) { @@ -3032,9 +3034,10 @@ private bool TryRelocateXRefSection(long offset, XRefSectionRecord section, out var readThrough = _chainValuesCut?.Count ?? 0; var state = TryReadXRefSection(candidate, attempt, out previous, out hybrid, candidate: true); - // A section a guard kept from being read is the section all the same: found where it was moved, it is where a - // guard cut the chain, as where it is named (#263). Any other place is dropped with what it read through. - if (state == XRefSectionState.Read || (state == XRefSectionState.Malformed && attempt.CutByLimit)) + // A section a value it is read with kept from being read, leading to an object a guard cut, is the section all the + // same: found where it was moved, it is where a guard cut the chain, as where it is named (#263). A place a guard + // cut the dictionary of says nothing of what is there, and is dropped as any other is, with what it read through. + if (state == XRefSectionState.Read || (state == XRefSectionState.Malformed && attempt.CutByValue)) { section.RelocateTo(attempt); _diagnostics.Repair( @@ -3722,8 +3725,8 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect out var number, out var syntaxFault, XRefWindow, - candidate ? DeclaresCrossReferenceStream : null) || - !DeclaresCrossReferenceStream(value)) + candidate ? DeclaresOnlyCrossReferenceStream : null) || + !(candidate ? DeclaresOnlyCrossReferenceStream(value) : DeclaresCrossReferenceStream(value))) { if (!candidate && _guardsReached != guards && value is PdfDictionary { } cut && !cut.ContainsKey(PdfName.Type)) { @@ -3785,6 +3788,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect { section.Fault = lengthFault; section.CutByLimit = true; + section.CutByValue = true; (_chainValuesCut ??= []).Add(past); return XRefSectionState.Malformed; } @@ -3807,6 +3811,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect if (unreadCut is { } cut) { section.CutByLimit = true; + section.CutByValue = true; (_chainValuesCut ??= []).Add(cut); } @@ -4238,6 +4243,15 @@ private static bool DeclaresCrossReferenceStream(PdfObject value) => (value is PdfStream stream ? stream.Dictionary : value as PdfDictionary) is { } dictionary && (dictionary.IsOfType(PdfName.XRef) || dictionary.ContainsKey(PdfName.W)); + /// + /// Determines whether what a place a section may have been moved to holds declares itself a cross-reference stream, and + /// no other type: /Type /XRef, or a /W beside no /Type that names another. A guess is no section + /// where it says it is something else — a CIDFont's /W gives its glyphs' widths. + /// + private static bool DeclaresOnlyCrossReferenceStream(PdfObject value) => + (value is PdfStream stream ? stream.Dictionary : value as PdfDictionary) is { } dictionary && + (dictionary.IsOfType(PdfName.XRef) || (dictionary.ContainsKey(PdfName.W) && dictionary.GetRaw(PdfName.Type) is not PdfName)); + /// /// Says which value a cross-reference stream's dictionary refers to, among those its rows are read with, that no section /// read so far places where it can be read; null when every one can be. diff --git a/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs b/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs index 64a8466b..60fa0c4d 100644 --- a/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs +++ b/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs @@ -67,6 +67,13 @@ public XRefSectionRecord(string namedBy, long namedOffset, long namedFrom) /// public bool CutByLimit { get; set; } + /// + /// Gets a value indicating whether what cut the section is a value it is read with — one its rows are read with, or its + /// /Length — that leads to an object one of the reader's guards cut: the section was found whole, and is the + /// guard's to keep from being read (#263). A guard that cut the section's own dictionary leaves it unknown. + /// + public bool CutByValue { get; set; } + /// /// Gets a value indicating whether cut the section's /// trailer: what lay past the cut — an older section's /Prev, a key a newer trailer gives — is unknown. @@ -130,6 +137,7 @@ public void RelocateTo(XRefSectionRecord attempt) RefusedRow = attempt.RefusedRow; Incomplete = attempt.Incomplete; CutByLimit = attempt.CutByLimit; + CutByValue = attempt.CutByValue; TrailerCutByLimit = attempt.TrailerCutByLimit; ChainHiddenByGuard = attempt.ChainHiddenByGuard; ChainHiddenAtStream = attempt.ChainHiddenAtStream; diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index adc5173d..25f4f89f 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -2384,11 +2384,13 @@ public void A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_bef [Theory] [InlineData("<< /A 1 /B >>", 64 * 1024)] [InlineData("<< /Producer (an object longer than the trailer guard allows, which a place the section was moved to is only a guess at; it is read as itself when it is asked for, under the guard that bounds objects) >>", 128)] + [InlineData("<< /Type /Font /Subtype /CIDFontType2 /BaseFont /Foo /DW 1000 /W [0 [500] 1 [500] 2 [500] 3 [500] 4 [500] 5 [500] 6 [500] 7 [500] 8 [500] 9 [500] 10 [500] 11 [500] 12 [500] 13 [500] 14 [500] 15 [500] 16 [500] 17 [500] 18 [500] 19 [500] 20 [500] 21 [500] 22 [500] 23 [500] 24 [500] 25 [500] 26 [500] 27 [500] 28 [500] 29 [500]] >>", 128)] public void A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading(string six, int maxTrailerLength) { // /Prev names a place 2 bytes into object 6, just before the older stream: object 6, nearer, is tried first, and is - // no section — a dictionary with a key short of a value, or one the trailer guard cuts. Neither its syntax fault - // nor the guard is reported: it was read as a section, and is read as itself when asked for. + // no section — a dictionary with a key short of a value, or one the trailer guard cuts, a CIDFont's whose /W gives + // its glyphs' widths among them. Neither its syntax fault nor the guard is reported: it was read as a section, and + // is read as itself when asked for. The font, once taken for the moved section, ended the chain as the guard's. var built = ChainFiles.UnderAnUpdate("/Type /XRef /Size 8 /W [1 4 2] /Length {length}", "0", compressed: false, predicted: false, placed: true, six: six); var stream = PdfTemplate.OffsetOf(built, "\n7 0 obj") + 1; var named = PdfTemplate.OffsetOf(built, "\n6 0 obj") + 3; @@ -2405,6 +2407,29 @@ public void A_place_a_section_may_have_moved_to_that_holds_another_object_leaves document.GetObject(new PdfObjectId(6)).Should().NotBeSameAs(PdfNull.Instance); } + [Fact] + public void A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at() + { + // Object 6 is a CIDFont that writes its /Type past a /W the trailer guard cuts: cut before it says what it is, the + // dictionary may be a section's, and the guard that hid it is reported. It is not the section all the same — only a + // value a section is read with that leads to an object a guard cut is (#263) —: the older stream, farther, is found. + var built = ChainFiles.UnderAnUpdate("/Type /XRef /Size 8 /W [1 4 2] /Length {length}", "0", compressed: false, predicted: false, placed: true, six: "<< /Subtype /CIDFontType2 /W [0 [500] 1 [500] 2 [500] 3 [500] 4 [500] 5 [500] 6 [500] 7 [500] 8 [500] 9 [500] 10 [500] 11 [500] 12 [500] 13 [500] 14 [500] 15 [500] 16 [500] 17 [500] 18 [500] 19 [500] 20 [500] 21 [500] 22 [500] 23 [500] 24 [500] 25 [500] 26 [500] 27 [500] 28 [500] 29 [500]] /Type /Font >>"); + var stream = PdfTemplate.OffsetOf(built, "\n7 0 obj") + 1; + var named = PdfTemplate.OffsetOf(built, "\n6 0 obj") + 3; + var file = Encoding.Latin1.GetBytes(Encoding.Latin1.GetString(built).Replace( + string.Create(CultureInfo.InvariantCulture, $"/Prev {stream}"), + string.Create(CultureInfo.InvariantCulture, $"/Prev {named}"), + StringComparison.Ordinal)); + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxTrailerLength = 128 } }; + + using var document = PdfDocument.Open(file, options); + + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(diagnostic => diagnostic.Code).Should().Equal(PdfDiagnosticCodes.LimitTrailer, PdfDiagnosticCodes.XRefOffsetAdjusted); + document.Diagnostics[1].Message.Should().EndWith(string.Create(CultureInfo.InvariantCulture, $"was found {stream - named} bytes from there.")); + document.GetObject(new PdfObjectId(6)).AsDictionary().IsOfType(PdfName.Font).Should().BeTrue(); + } + [Fact] public void Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with() { From 2b5e0d8fa3db549de4686cbbce8e5c26e5d2a011 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 10:31:57 +0000 Subject: [PATCH 092/146] fix: make room in the index by a /Size as written, reading no reference through for it A cross-reference stream's /Size beside an /Index is left as written, since it numbers no row. When that stream is the newest section, its dictionary becomes the index's trailer, and each older section made room by resolving its /Size: an object a guard cut after a whole reference was read through, loaded at opening, and its guard reported or thrown. Room is now made by a /Size written direct, as ISO 32000-1 makes it; a reference gives none. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 6 ++- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 40 +++++++++++++++++++ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 17 ++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index d7edcf4d..04870f7f 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -4696,7 +4696,7 @@ private static int RowsHeld(ReadOnlySpan rows) => /// /// Makes room in the index for the a section is about to give, so that the map is sized once /// rather than grown by doubling, and for no more numbers than lets it - /// take from here on, nor than the newest section's /Size numbers once it is read: one more than the document's + /// take from here on, nor than the newest section's /Size numbers, written direct, once it is read: one more than the document's /// highest object number (ISO 32000-1, 7.5.5), which an older section's rows, superseded, add nothing to. The caller /// bounds the rows by those the section holds: a count the file declares is a claim. /// @@ -4711,7 +4711,9 @@ private void ReserveIndex(long rows) var most = _guard.Bound(PdfLimit.IndexEntries); var room = (int)Math.Min(rows, int.MaxValue); - if (_xref.Trailer.GetInteger(PdfName.Size) is { } size) + // The /Size as written, which ISO 32000-1 makes direct (Table 15, 7.5.8.2): a reference is not read through to make + // room by, as the newest cross-reference stream's beside an /Index, which numbers no row, would be while the chain is. + if (_xref.Trailer.GetRaw(PdfName.Size) is PdfInteger { Value: var size }) { // A size the index holds as many entries as already asks for no room, nor does one below zero, which is not // narrowed to an int that would. One the sections read so far number past is stale, and they stand for it. diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 3abda3e9..8f800fe6 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -1095,6 +1095,46 @@ public static byte[] StreamUpdateHidingItsHighestNumber(int padding, bool real = return file.ToArray(); } + /// + /// A file of two revisions. The first indexes the catalog, the page tree and the page, in a classic table or, with + /// , in cross-reference stream 4. The second writes object 5, 9 0 R followed by + /// 1,100 spaces, and object 9, 10, and indexes them in cross-reference stream 7, the newest section, whose + /// /Size names object 5 beside an /Index the rows are numbered by. A reader that reads object 5 whole finds + /// a valid file but for that indirect /Size. + /// + public static byte[] StreamSizedThroughAReferenceOverAnOlderSection(bool olderStream) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + var first = file.Position; + + if (olderStream) + { + var rows = Encode([(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), (1, first, 0)], Rows.Raw); + file.Stream(4, string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size 5 /W [1 4 2] /Root 1 0 R /Length {rows.Length}"), rows); + } + else + { + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"xref\n0 4\n0000000000 65535 f\r\n{file.OffsetOf(1):D10} 00000 n\r\n{file.OffsetOf(2):D10} 00000 n\r\n{file.OffsetOf(3):D10} 00000 n\r\ntrailer\n<< /Size 4 /Root 1 0 R >>\n")); + } + + file.StartXRef(first); + file.Object(5, "9 0 R" + new string(' ', 1100)); + file.Object(9, "10"); + var stream = file.Position; + var data = Encode([(1, file.OffsetOf(5), 0), (1, stream, 0), (1, file.OffsetOf(9), 0)], Rows.Raw); + file.Stream( + 7, + string.Create(CultureInfo.InvariantCulture, $"/Type /XRef /Size 5 0 R /Index [5 1 7 1 9 1] /W [1 4 2] /Root 1 0 R /Prev {first} /Length {data.Length}"), + data); + file.StartXRef(stream); + return file.ToArray(); + } + /// /// A file of two classic tables. The first indexes the catalog, the page tree, the page and object 12, the highest /// number; the second, an update, writes the catalog, the page tree and the page again, indexes them with object 4, a diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index b8dce85d..735d80bd 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1280,6 +1280,23 @@ public void A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whol document.Reader.Structure.Sections.Should().HaveCount(2).And.OnlyContain(section => section.State == IO.XRef.XRefSectionState.Read); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void A_size_beside_an_index_in_the_newest_section_is_not_read_to_make_room_for_an_older_section_s_rows(bool olderStream) + { + // As above, stream 7 the newest section, over an older table or stream: its dictionary, the index's trailer, gave the + // /Size the index made room by for the older section's rows, read through to object 5 and loaded at opening. Room is + // made by a /Size as written, which ISO 32000-1 makes direct (Table 15, 7.5.8.2): a reference gives none. + var file = ChainFiles.StreamSizedThroughAReferenceOverAnOlderSection(olderStream); + + using var document = PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); + + document.Diagnostics.Should().BeEmpty(); + document.Reader.Structure.Sections.Should().HaveCount(2).And.OnlyContain(section => section.State == IO.XRef.XRefSectionState.Read); + document.GetObject(new PdfObjectId(3)).AsDictionary().IsOfType(PdfName.Page).Should().BeTrue(); + } + [Fact] public void Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value() { From 34cc6c8b422cd5ddb95f1040a0805dc28341d8ef Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 10:34:23 +0000 Subject: [PATCH 093/146] fix: report a section found away from where it is named though a guard kept it unread A section relocated as the guard's has its state set back to malformed, so that the chain ends there; xref.section-shifted looked for the relocated state alone, and lost the file's own fault, an offset that names the wrong place, which the same file read whole reports. The rule now asks whether the section was found away from where it is named, whatever became of it there. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs | 6 ++++++ src/AdCodicem.Pdf/Validation/Rules/SectionShiftedRule.cs | 4 ++-- tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 3 +++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs b/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs index 60fa0c4d..ff16fed7 100644 --- a/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs +++ b/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs @@ -43,6 +43,12 @@ public XRefSectionRecord(string namedBy, long namedOffset, long namedFrom) /// Gets where the section was read, which is where it was named unless it was relocated. public long Offset { get; private set; } + /// + /// Gets a value indicating whether the section was found away from where it was named, whatever became of it there: + /// read, or kept from being read by a guard, which leaves its state . + /// + public bool WasRelocated => Offset != NamedOffset; + /// Gets what became of the section. public XRefSectionState State { get; set; } diff --git a/src/AdCodicem.Pdf/Validation/Rules/SectionShiftedRule.cs b/src/AdCodicem.Pdf/Validation/Rules/SectionShiftedRule.cs index 7d4e05c5..9909013e 100644 --- a/src/AdCodicem.Pdf/Validation/Rules/SectionShiftedRule.cs +++ b/src/AdCodicem.Pdf/Validation/Rules/SectionShiftedRule.cs @@ -1,5 +1,4 @@ using System.Globalization; -using AdCodicem.Pdf.IO.XRef; namespace AdCodicem.Pdf.Validation.Rules; @@ -26,7 +25,8 @@ public void Check(ValidationContext context) { foreach (var section in context.Structure.Sections) { - if (section.State != XRefSectionState.Relocated) + // Found there and read, or found there and kept from being read by a guard, the section's own fault (#263). + if (!section.WasRelocated) { continue; } diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 735d80bd..d19d0afe 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1918,6 +1918,9 @@ public void A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_wa moved.CutByLimit.Should().BeTrue(); var findings = new PdfValidator().Validate(document).Findings.Select(finding => finding.RuleId); findings.Should().Contain(PdfValidationRuleIds.XRefCheckedInPart).And.NotContain(PdfValidationRuleIds.XRefSectionNotFound); + + // The /Prev names the wrong offset whatever the guard did: the section moved is reported, as read whole. + findings.Should().Contain(PdfValidationRuleIds.XRefSectionShifted); } [Fact] From 7b19e78b80cf63136eca10420f39de4b4ebdb906 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 10:34:23 +0000 Subject: [PATCH 094/146] test: hold an object stream member whose R ends where a guard cuts the data What follows the R would say whether it ends a reference; at the end of data a guard cut there is nothing to ask, and the member stays cut. No test reached that bound: without it, reading the member threw an IndexOutOfRangeException, and every test passed. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../AdCodicem.Pdf.Tests/ReaderLimitsTests.cs | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs b/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs index 9b5c8407..87978863 100644 --- a/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs @@ -958,6 +958,41 @@ public void A_member_whose_array_ends_where_a_guard_cuts_the_data_is_read_whole( document.Diagnostics.Select(diagnostic => diagnostic.Code).Should().Equal(PdfDiagnosticCodes.LimitDecodedStream); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void A_member_whose_R_ends_where_a_guard_cuts_the_data_is_cut(bool throwOnLimit) + { + // Member 10, "6 0 R", ends on byte 10, where MaxDecodedStreamLength cuts the data: what follows the R, which would + // say whether it ends a reference, is the guard's, and nothing past the data is asked. The member reads as the + // number before it, cut, or the guard is thrown. + const string Data = "10 0 6 0 R (more data) "; + var hex = Convert.ToHexString(Encoding.Latin1.GetBytes(Data)) + ">"; + var written = new HandWrittenFile(version: "1.5"); + var rows = new Dictionary + { + [1] = (1, written.Object(1, Catalog), 0), + [2] = (1, written.Object(2, Pages), 0), + [3] = (1, written.Object(3, string.Create(CultureInfo.InvariantCulture, $"<< /Type /ObjStm /N 1 /First 5 /Filter /ASCIIHexDecode /Length {hex.Length} >>\nstream\n{hex}\nendstream")), 0), + [10] = (2, 3, 0), + }; + written.CrossReferenceStream(12, 13, rows, "/Root 1 0 R"); + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxDecodedStreamLength = 10 }, ThrowOnLimit = throwOnLimit }; + + using var document = PdfDocument.Open(written.Bytes, options); + + if (throwOnLimit) + { + var reading = () => document.GetObject(new PdfObjectId(10)); + reading.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxDecodedStreamLength)); + return; + } + + document.GetObject(new PdfObjectId(10)).Should().BeOfType().Which.Value.Should().Be(6); + document.Reader.IsCutAtLimit(10).Should().BeTrue(); + document.Diagnostics.Select(diagnostic => diagnostic.Code).Should().Equal(PdfDiagnosticCodes.LimitDecodedStream); + } + [Fact] public void A_member_left_open_where_a_guard_cuts_the_data_at_the_next_member_s_start_is_the_file_s_fault() { From 846df96e07f424f135308470b87be84a25068631 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 10:36:22 +0000 Subject: [PATCH 095/146] fix: judge by Table 8 only the parameters a cross-reference stream's /DecodeParms gives A /Filter, or an element of one, that is a dictionary ends decoding, and the data is left as it is: decoding reads nothing it holds. file.trailer-value-wrong walked it as it walks a /DecodeParms, and reported a /Predictor or /Columns written as a real where Table 8 asks for an integer, a parameter the reader never read. Table 8 now governs a /DecodeParms alone. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../Validation/Rules/TrailerValueWrongRule.cs | 14 ++++++++------ .../CrossReferenceRuleTests.cs | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs b/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs index 7d00cdd4..9948a73a 100644 --- a/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs +++ b/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs @@ -203,8 +203,9 @@ public void Integers(PdfDictionary trailer, PdfName key) /// /// Reports a cross-reference stream's /Filter or /DecodeParms written as a reference, and, in a file - /// that declares PDF 2.0, each element and entry written as one; and the integers of Table 8 written as reals. A - /// reference among them is followed where decoding follows it, as says. + /// that declares PDF 2.0, each element and entry written as one; and the integers of Table 8 a /DecodeParms + /// gives written as reals — not a dictionary a /Filter gives, which ends decoding unread. A reference among + /// them is followed where decoding follows it, as says. /// public void Decoding(PdfDictionary trailer, bool pdf20, DecodingReads reads) { @@ -214,6 +215,7 @@ public void Decoding(PdfDictionary trailer, bool pdf20, DecodingReads reads) { var name = FileQuote.Name(key); var value = trailer.GetRaw(key); + var decodeParms = key == PdfName.DecodeParms; Value("its " + name, value, Iso1, integer: null); switch (reads.Resolve(value)) @@ -226,24 +228,24 @@ public void Decoding(PdfDictionary trailer, bool pdf20, DecodingReads reads) if (reads.Resolve(array[i]) is PdfDictionary parameters) { - Parameters(parameters, what, inside, reads); + Parameters(parameters, what, inside, reads, decodeParms); } } break; case PdfDictionary parameters: - Parameters(parameters, "its " + name, inside, reads); + Parameters(parameters, "its " + name, inside, reads, decodeParms); break; } } } - private void Parameters(PdfDictionary parameters, string where, string? inside, DecodingReads reads) + private void Parameters(PdfDictionary parameters, string where, string? inside, DecodingReads reads, bool decodeParms) { foreach (var (key, value) in parameters) { - var integer = Array.IndexOf(IntegerParameters, key) >= 0 ? "Table 8" : null; + var integer = decodeParms && Array.IndexOf(IntegerParameters, key) >= 0 ? "Table 8" : null; Value(FileQuote.Name(key) + " of " + where, value, inside, integer, reads.Follows(value)); } } diff --git a/tests/AdCodicem.Pdf.Tests/CrossReferenceRuleTests.cs b/tests/AdCodicem.Pdf.Tests/CrossReferenceRuleTests.cs index c1e21ce7..d430d982 100644 --- a/tests/AdCodicem.Pdf.Tests/CrossReferenceRuleTests.cs +++ b/tests/AdCodicem.Pdf.Tests/CrossReferenceRuleTests.cs @@ -1611,6 +1611,21 @@ public void A_predictor_written_as_a_real_in_an_array_of_parameters_is_reported( "gives /Predictor of element 0 of its /DecodeParms as a real number, where Table 8 of ISO 32000-1 asks for an integer."); } + [Theory] + [InlineData("/Filter 5 0 R", "<< /Predictor 12.0 /Columns 7.0 >>")] + [InlineData("/Filter [5 0 R]", "<< /Predictor 12.0 /Columns 7.0 >>")] + [InlineData("/Filter << /Predictor 12.0 /Columns 7.0 >>", "0")] + [InlineData("/Filter [<< /Predictor 12.0 /Columns 7.0 >>]", "0")] + public void Parameters_a_filter_gives_as_a_dictionary_are_not_judged_by_what_decode_parameters_ask(string filter, string five) + { + // A /Filter, or an element of one, that is a dictionary ends decoding, the data left as it is: decoding reads nothing + // it holds, which Table 8, a /DecodeParms', does not govern. Its reals were reported as Table 8 asks (#263). + var file = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] " + filter + " /Length {length}", five, compressed: false, predicted: false, placed: true); + + Validate(file).Findings.Select(finding => finding.Message).Should().NotContain(message => message.Contains("Table 8", StringComparison.Ordinal)); + } + [Fact] public void The_entry_a_deferred_length_found_off_its_object_is_judged_as_the_file_wrote_it() { From 3bdd725e4456a862c4ed7cce95da87ecb6abc30e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 10:36:44 +0000 Subject: [PATCH 096/146] test: give the proof of an endstream as why the data a spelled one ends is the guard's Two comments said the data was the guard's because its object's endobj did not follow the endstream the data spells. Whether an endobj follows no longer decides it: nothing past the object's end proves that endstream the stream's own. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index d19d0afe..bef2f5a6 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -751,8 +751,9 @@ public void Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_t [InlineData("left out")] public void An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s(string guard) { - // The content stream's data spells endstream inside a string, which its object's endobj does not follow; its own - // endstream and endobj come after the 40 bytes. Read whole, the length passes the spelled one. Where a guard cut the + // The content stream's data spells endstream inside a string, which nothing that would prove it the stream's own + // follows — an object the index places, the end of the file, xref or startxref —; its own endstream and endobj come + // after the 40 bytes. Read whole, the length passes the spelled one. Where a guard cut the // /Length, or left it out of the index, the search stops there, and the object was said to lack its endobj: the // length that tells the two apart is the guard's, and so is the data up to it (#224). const string Data = "BT /F1 12 Tf 72 712 Td (endstream) Tj ET"; @@ -1510,8 +1511,9 @@ public void Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chai section.CutByLimit.Should().BeTrue(); section.Incomplete.Should().BeFalse(); - // Loaded, the stream ends at the endstream its rows spell, which its object's endobj does not follow: the data is the - // guard's, and the object was said to lack its endobj. + // Loaded, the stream ends at the endstream its rows spell, which nothing past its object's end proves its own — an + // object the index places, the end of the file, xref or startxref —: the data is the guard's, and the object is not + // said to lack its endobj, which is the guard's to hide. document.GetObject(new PdfObjectId(7)).AsStream().Required().Data.CutByGuard.Should().BeTrue(); document.Reader.IsEndObjMissing(7, out _).Should().BeFalse(); From 0733c3522b326985777ed75a832e6218324ee52e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 10:40:25 +0000 Subject: [PATCH 097/146] test: hold what the proofs of an endstream read under a hidden length below the file The proof that an endstream found under a hidden /Length is the stream's own reads past it, and counts toward what the searches may read; the bound's reason named three ways past the window only. A file of 400 one-byte streams sharing a /Length a guard cut now has its proofs read 48 bytes each, less than the file, every endstream proven. The bound's reason, ADR 34, the reference, the concepts and the threat model name the proof as a fourth way, the file once more. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- ...-is-readable-and-the-readers-guards-are.md | 4 +- docs/threat-model.md | 2 +- docs/website/docs/concepts/diagnostics.md | 4 +- docs/website/docs/reference/reader-limits.md | 2 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 17 ++++++--- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 37 +++++++++++++++++++ 6 files changed, 57 insertions(+), 9 deletions(-) diff --git a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md index 40ec5055..b67e3e19 100644 --- a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md +++ b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md @@ -100,7 +100,9 @@ bounded by `EndStreamSearchPasses`, four times the file for a document's searche constant with its reason rewritten: a valid file is read past the parser's window only by the search for the data of a cross-reference stream whose `/Length` the chain cannot read, by the search for the data of a stream whose `/Length` a guard cut or left out of the index, and by white space and comments after a declared length that run past a first -read — about twice the file at most. `MaxEndStreamGap`, the four bytes of white space accepted +read — about twice the file at most; the proof that an `endstream` found under a hidden `/Length` is the stream's +own reads 48 bytes past each, about the file once more, three times it in all (amended on 2026-10-24). +`MaxEndStreamGap`, the four bytes of white space accepted between a stream's data and its `endstream`, is removed: more, or a comment, made a right `/Length` a wrong one, and white space and comments are read on as far as they run, counted with the searches — a comment only once white space comes before it, since a `%` right where the length ends is the data's. Comments after the `endobj` that ends a stream diff --git a/docs/threat-model.md b/docs/threat-model.md index 0e5f48c8..600f9ca0 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -355,7 +355,7 @@ Known gaps: | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | | A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places, the end of the file, or `xref` or `startxref`, which data that goes on is never followed by: read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | -| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | +| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 bytes past each, about the file once more; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | | Encoded data of any length: a `/Length` of a few digits claiming gigabytes, one past what an array holds, which would exhaust memory at each read, or data decoding hands back as it is, which `MaxDecodedStreamLength` does not bound | `MaxStreamLength` (*The guards*): what is read of one stream's data at once stops at it, what lies past it is cut by a guard and reported as `limit.stream-length` — once for the stream in its document's diagnostics, and with each decoding that meets the cut in diagnostics a caller passes to `Decode`, as `limit.decoded-stream` is —, and the filters decode what was read as data the reader cut ([#170]) | `ReaderLimitsTests.A_stream_cut_at_MaxStreamLength_decodes_what_lies_within_it_as_data_the_reader_cut_and_is_reported_once`, `Data_decoding_hands_back_as_it_is_is_bounded_by_MaxStreamLength_and_not_by_MaxDecodedStreamLength`, `A_length_past_what_an_array_holds_ends_at_the_guard_and_never_in_an_exhausted_memory`, `Decoding_into_diagnostics_of_one_s_own_reports_the_cut_max_stream_length_made_each_time` | No | diff --git a/docs/website/docs/concepts/diagnostics.md b/docs/website/docs/concepts/diagnostics.md index 41aea26f..64c5dac6 100644 --- a/docs/website/docs/concepts/diagnostics.md +++ b/docs/website/docs/concepts/diagnostics.md @@ -97,7 +97,9 @@ was read before it — though a damaged index can still serve another copy of an what was read first. A file crafted for it can still make what a stream holds depend on which object was asked for first. What the searches of one document read together is bounded: none starts once they have read four times the file's length. For a valid file, what they read — its cross-reference streams searched, its streams whose length a guard hid, -the white space after a declared length read on — comes to about twice the file at most, and the searches of damaged streams read twice the file at most — 0.35 +the white space after a declared length read on — comes to about twice the file at most, and the proofs that an `endstream` +found under a hidden length is the stream's own, 48 bytes past each, to about the file once more; the searches of damaged +streams read twice the file at most — 0.35 of it at most, all searches together, in the test corpus — unless their stretches overlap: one object's header inside another's dictionary, or a header the search does not take — a regular character glued before its number — that the index as the file wrote it does not place. Past the bound, only the bytes the parser holds are searched: a declared diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index 74e16829..6dc40685 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -189,7 +189,7 @@ the documents producers write. They are not options: |---|---|---| | Nesting of arrays and dictionaries | 128 levels | Real documents nest a handful of levels; the bound keeps a hostile file from exhausting the stack | | A reference that resolves to another reference | followed 32 times | Producers write the object itself; the bound breaks a chain that loops | -| Searches for a stream's `endstream` | four times the file's length, for a document's searches together | A valid stream's `endstream` follows its declared length: the file is read past the bytes an object is parsed through only for the data of a cross-reference stream whose `/Length` the chain cannot read, for the data of a stream whose `/Length` a guard cut or left out of the index, and for white space and comments after a declared length that run past a first read — about twice the file at most. Past the bound, a stream whose `endstream` the bytes read with its object do not show is not searched for further, and its report says so | +| Searches for a stream's `endstream` | four times the file's length, for a document's searches together | A valid stream's `endstream` follows its declared length: the file is read past the bytes an object is parsed through only for the data of a cross-reference stream whose `/Length` the chain cannot read, for the data of a stream whose `/Length` a guard cut or left out of the index, and for white space and comments after a declared length that run past a first read — about twice the file at most —, and the proof that an `endstream` found under a hidden `/Length` is the stream's own reads 48 bytes past each, about the file once more. Past the bound, a stream whose `endstream` the bytes read with its object do not show is not searched for further, and its report says so | | Objects loaded while loading another | 64 | An indirect `/Length` leads to an integer, and a stream resolves its filters and the parameters they read before decoding any of its data: no level holds decoded data while the next loads | | Object numbers | up to 2,147,483,647 | An object's number is an `int`, and ISO 32000-1's Annex C advises at most 8,388,607 indirect objects. A number past it is refused where it is read — a header, a reference, a subsection, a row, an object stream's header —, never narrowed to one that would name another object. An object is numbered from 1; 0 names the head of the free list, where a reference, a subsection or a row may give it | | An object stream's members | each read to the next one's start, the last to the end of the decoded data | ISO 32000-1 (7.5.7) has a header list its objects by increasing offset, so that each lies before the next one starts: a pair whose offset is not past the one before it, or whose object would start past the data, ends the header there, as a stray token does, and the objects listed before it are read. A member past `MaxObjectLength` is that limit's | diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 04870f7f..e09dac6e 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -152,11 +152,12 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// of different streams lies apart, and so do the runs of white space after them: together those reads cover the file /// once, and one read past each — about twice the file at most. A fourth adds a little to each stream of the second /// way: the proof that an endstream found is the stream's own (#224) reads past it to its object's end and what - /// follows, three first reads of bytes, which double only as white space runs on — a few times - /// a stream of a few bytes, so that a file of such streams alone reads about three times itself. Every other search is of an invalid stream, whose - /// endstream does not follow its declared length. Each - /// reads from a stream's data to the first object header the bytes hold after it, or to the next object the index as - /// written places, whichever is nearer, and its reads grow from twice what the parser holds of the data + /// follows, three first reads of bytes, which double only as white space runs on — 48 bytes, + /// about what the smallest such stream spans with its header, so that the proofs add the file once more at most: three + /// times it in all, within the four the searches may read. Every other search is of an invalid stream, whose + /// endstream does not follow its declared length. Each reads from a stream's data to the first object header the + /// bytes hold after it, or to the next object the index as written places, whichever is nearer, and its reads grow + /// from twice what the parser holds of the data /// (): it reads at most about twice what its object spans, and the searches of /// objects whose headers follow one another read disjoint stretches — the file twice over at most. Only searches that /// share a stretch read more: objects that overlap, the header of one inside the dictionary of another, before its @@ -1452,6 +1453,12 @@ private static bool Consume(HeaderReading reading, ref long consumed, long end) /// internal bool IsCutAtLimit(int number) => _cutAtLimit.Contains(number); + /// + /// Gets how many bytes the searches for endstream, and the proofs that what they found is a stream's own, have + /// read so far: what bounds. + /// + internal long EndStreamSearchBytes => _endStreamSearchBytes; + /// /// Determines whether the object leads to — through a chain of references too, as /// follows it — is one a guard cut, or left out of the index: what it holds may lie diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index bef2f5a6..36b71846 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -949,6 +949,43 @@ public void An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_len guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries); } + [Fact] + public void The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file() + { + // 400 streams of a byte share a /Length, object 9, which MaxObjectLength cuts before its value: each one's endstream is + // found among the bytes the parser holds, and proven its own by the next object the index places, a few bytes past + // its endobj. The proofs read those few bytes, which count toward what the searches may read, four times the file: a + // file of such streams has them read less than itself (#224). + const int Streams = 400; + var file = new HandWrittenFile(version: "1.5"); + var rows = new Dictionary + { + [1] = (1, file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), 0), + [2] = (1, file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), 0), + [3] = (1, file.Object(3, Page), 0), + [9] = (1, file.Object(9, new string(' ', 2000) + "1"), 0), + }; + + for (var number = 10; number < 10 + Streams; number++) + { + rows[number] = (1, file.Object(number, "<< /Length 9 0 R >>\nstream\nq\nendstream"), 0); + } + + file.CrossReferenceStream(10 + Streams, 11 + Streams, rows, "/Root 1 0 R"); + var bytes = file.Bytes; + + using var document = PdfDocument.Open(bytes, ObjectGuard); + + for (var number = 10; number < 10 + Streams; number++) + { + var stream = document.GetObject(new PdfObjectId(number)).AsStream().Required(); + stream.Data.CutByGuard.Should().BeFalse(Invariant($"stream {number}'s endstream is proven its own")); + stream.Data.Length.Should().Be(1); + } + + document.Reader.EndStreamSearchBytes.Should().Be(Streams * 48, "each proof reads three times 16 bytes").And.BeLessThan(bytes.Length); + } + [Theory] [InlineData("Rx", false)] [InlineData("RG", false)] From 252443285447224f6d03db33fb4a2a5b5fa2f3d4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 10:42:38 +0000 Subject: [PATCH 098/146] test: pin that the offset a section is named at keeps what it read through What the offset the chain names holds is read as a load reads it, and keeps all of its reading though it holds no section: its reports, the guards it reached, and what it read through, loaded once the chain is read. Only a place guessed near it leaves nothing, which the record of what the chain read through said of any object read as a section and refused. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 5 ++-- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 26 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index e09dac6e..530e802a 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -296,8 +296,9 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// /// The objects a guard cut that a cross-reference stream's values led to, which the chain could not read and ended at, and - /// those it read through, the cut having left their value a whole reference — but for an object it read as a section - /// and refused —: each is loaded once the chain is read, so that its guard is reached where a load reaches it (#263). + /// those it read through, the cut having left their value a whole reference — but for an object it read at a place the + /// section may have been moved to, and refused; what the offset the chain names holds keeps all of its reading —: each + /// is loaded once the chain is read, so that its guard is reached where a load reaches it (#263). /// Let go then, or when the chain is rebuilt. /// private List? _chainValuesCut; diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 36b71846..65f5edf2 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1298,6 +1298,32 @@ public void An_object_read_through_for_a_place_that_holds_no_section_reaches_the } } + [Fact] + public void What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would() + { + // As above, the /Prev naming object 8's header exactly: the place the chain names is read as a load reads it, and + // keeps all of its reading though it holds no section — its stream's length reported, and object 5, read through, + // loaded once the chain is read, the guard reported, or thrown. Only a place guessed near it leaves nothing (#263). + var file = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] /Length {length}", + "6 0 R" + new string(' ', 1100), + compressed: false, + predicted: false, + placed: true, + placedToo: new Dictionary { [6] = "8", [8] = "<< /Length 5 0 R >>\nstream\nabc\nendstream" }); + var text = Encoding.Latin1.GetString(file); + var stream = text.IndexOf("7 0 obj", StringComparison.Ordinal); + var eight = text.IndexOf("8 0 obj", StringComparison.Ordinal); + file = Encoding.Latin1.GetBytes(text.Replace(Invariant($"/Prev {stream} "), Invariant($"/Prev {eight} "), StringComparison.Ordinal)); + + using var document = PdfDocument.Open(file, ObjectGuard); + document.Diagnostics.Select(entry => entry.Code).Should().Equal( + PdfDiagnosticCodes.StreamLengthInvalid, PdfDiagnosticCodes.XRefOffsetAdjusted, PdfDiagnosticCodes.LimitObject); + + var opening = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }).Dispose(); + opening.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); + } + [Fact] public void A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read() { From 7e58b708ebebd5bf9b1f161ed1e46f684bb58947 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 10:42:39 +0000 Subject: [PATCH 099/146] docs: name the spelled xref an endstream's proof accepts, and a /Size beside an /Index The pages said no data that goes on is followed by xref or startxref; data may spell them, and under a hidden /Length is then taken to end there, the risk the proof accepted, now named among the threat model's known gaps. ADR 34, the threat model, the concepts and the diagnostics reference listed a cross-reference stream's /Size beside an /Index among the values that end the chain; only a /Size with no /Index numbers rows, and only a /Prev or /XRefStm the reader follows ends it. The reference and the threat model say what a section's named offset keeps of its reading, and what a guessed place leaves. Refs #224, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- ...id-pdf-is-readable-and-the-readers-guards-are.md | 6 +++--- docs/threat-model.md | 13 ++++++++----- docs/website/docs/concepts/diagnostics.md | 7 ++++--- docs/website/docs/concepts/lazy-reading.md | 4 ++-- docs/website/docs/concepts/validation.md | 5 +++-- docs/website/docs/reference/diagnostics.md | 2 +- docs/website/docs/reference/reader-limits.md | 8 +++++--- 7 files changed, 26 insertions(+), 19 deletions(-) diff --git a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md index b67e3e19..731c29f0 100644 --- a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md +++ b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md @@ -202,9 +202,9 @@ guard: active by default, reported when it is reached, and lifted by an option. property is raised: the index is not rebuilt for them, which turned a sound file into an `xref.rebuilt` error (ADR 45) and could lose what the chain had read. A classic table cut by its length keeps its trailer. The chain ends at a cut — a table's, a stream's rows, a section or a trailer cut wherever the cut falls, a cross-reference stream - whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` leads to an object `MaxObjectLength` cut, or whose - `/Length` does where only it could say how far the data runs, a `/Prev` or an `/XRefStm` that does (amended on - 2026-10-23) —, since an older section, or its trailer, would stand for what the cut left out; and a section a guard + whose `/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, leads to an object `MaxObjectLength` + cut, or whose `/Length` does where only it could say how far the data runs, a `/Prev` or an `/XRefStm` the reader + follows that does (amended on 2026-10-23 and 2026-10-24) —, since an older section, or its trailer, would stand for what the cut left out; and a section a guard cut is where it was named, not looked for near there. One found near where it is named, whose values a guard keeps from being read, is the guard's there too (amended on 2026-10-24). A rebuild that damage forces on such a chain keeps the entries the chain read; one the chain cannot do without — a guard that leaves it no section — runs as before. diff --git a/docs/threat-model.md b/docs/threat-model.md index 600f9ca0..178dc250 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -115,9 +115,9 @@ leaves out of the index reads as null until it is raised, and the index is not r damage. A value read through a reference to an object a guard cut or left out of the index, through a chain of references too, is the guard's as well, whatever the cut left of it — null, or a token it parted —, and the reader judges none of it as the file's. That holds for a stream's `/Length` ([#224]) and the values its filters read -([#263]), for a cross-reference stream's `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` that -`MaxObjectLength` cut as the chain was read, and its `/Length` when no `endstream` ends its data, and for a `/Prev` or -`/XRefStm` that leads to such an object (*Indexing*); it adds no bound. When the document was opened with +([#263]), for a cross-reference stream's `/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, +that `MaxObjectLength` cut as the chain was read, and its `/Length` when no `endstream` ends its data, and for a `/Prev` +or `/XRefStm` the reader follows that leads to such an object (*Indexing*); it adds no bound. When the document was opened with `ThrowOnLimit`, reaching a guard throws `PdfLimitExceededException` instead. A stream read from a document decodes under that document's guards, however long after opening. `PdfReaderLimits` refuses a bound of zero or less and takes a length past `Array.MaxLength` as that length. Held by `ReaderLimitsTests` throughout; the @@ -282,7 +282,7 @@ Known gaps: | A classic table of any length | Its window grows ×4 from 64 KB up to `MaxXRefSectionLength`, and the rows read are kept. A table the guard cuts, or whose rows run past a full index, keeps its trailer, found past the rows left unread by a search a megabyte at a time (`ReadTrailerPastRows`): comments are skipped, and anything else no row holds is the section's fault, reported, the search going on to the `trailer` keyword, so that its `/Encrypt` still refuses the document. The search starts at the start of the line the cut fell in, outside any comment and past the table's own keyword; it reads each megabyte once, every byte of it looked at before the next is read, however many comments or stray bytes it holds, and is not made for a place a section may have been moved to, a guess. The chain ends there, since an older section's rows would stand for the objects the unread rows define. What those rows place reads as null until the guard is raised | `ReaderLimitsTests.A_table_whose_keywords_the_bound_cuts_reports_the_bound`, `A_table_its_length_bound_cuts_keeps_its_trailer_and_the_encryption_it_names`, `A_table_its_length_bound_cuts_ends_the_chain_though_its_trailer_names_an_older_section`, `What_ends_the_rows_a_length_bound_cut_is_judged_as_it_is_where_none_did`, `A_table_a_length_bound_cuts_that_runs_to_the_end_of_the_file_has_no_trailer`, `A_table_its_length_bound_cuts_keeps_the_encryption_its_trailer_names_past_what_no_row_holds`, `A_table_cut_before_its_keyword_ends_keeps_its_trailer`, `Rows_past_a_full_index_are_not_read_and_the_trailer_past_them_is`, `A_cut_in_a_comment_among_the_rows_searches_for_the_trailer_from_the_comment_s_start`, `The_trailer_past_rows_left_unread_is_found_wherever_the_search_s_window_ends`, `A_trailer_past_rows_left_unread_that_its_own_bound_cuts_keeps_what_precedes_the_cut`; `DocumentReaderTests.The_search_for_the_trailer_past_rows_left_unread_reads_the_file_once`; `CrossReferenceChainTests.Reads_an_object_a_length_bound_cut_out_of_its_table_as_null_until_the_guard_is_lifted`, `A_section_found_near_where_prev_names_it_is_not_searched_past_the_rows_a_guard_left_unread`; `WindowEdgeTests.A_classic_table_reads_whole_wherever_its_window_edge_falls_near_its_trailer` | No: seeds stay under 120 KB | | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). A hybrid file's stream's own `/Prev`, which the chain does not follow, going on through the table's, is not read at all, nor are the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place: neither names a section missing nor ends the chain for a guard, and the rule on trailer values judges them as written, a reference they hold not followed ([#263]). An object the guard cut after a whole reference it holds hides nothing: the chain reads through it, as a load does, and loads it once the chain is read. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s`, `A_table_an_xrefstm_names_in_a_stream_s_place_has_its_own_links_left_unread`, `A_prev_written_as_a_reference_to_a_real_is_judged_through_it_only_where_the_reader_follows_it` | Yes | -| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a section's offset that is no section leaves nothing of its reading, nor what it read through: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps — not for a place it dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); a place that declares another type — a CIDFont, whose `/W` gives widths —, or whose own dictionary a guard cut, is a guess dropped, the next place tried | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild` | No | +| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); a place that declares another type — a CIDFont, whose `/W` gives widths —, or whose own dictionary a guard cut, is a guess dropped, the next place tried | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild` | No | | A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, to the same bytes — not as data a guard cut, whose last group ASCII85 and ASCIIHex drop, which could take a decoding guard's cut away and charge the rows to the file —: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | @@ -354,7 +354,7 @@ Known gaps: | A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | -| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places, the end of the file, or `xref` or `startxref`, which data that goes on is never followed by: read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | +| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | | Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 bytes past each, about the file once more; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | @@ -374,6 +374,9 @@ Known gaps: - Accepted, with no issue: the last object of a classic file, whose stream lost its `endstream` and declares no length within the file the reader can take, takes as its data everything to the end of the file, its cross-reference table and trailer included: neither `xref` nor `trailer` stops the search ([#174]). +- Accepted, with no issue: under a `/Length` a guard cut or left out of the index, data that spells `endstream`, an + `endobj` or not, then `xref` or `startxref`, and goes on, ends there as the file's data, and its object as ending + there: only the hidden length could tell, and a valid file's last stream before its table is told so ([#224]). ### Decoding diff --git a/docs/website/docs/concepts/diagnostics.md b/docs/website/docs/concepts/diagnostics.md index 64c5dac6..6bdb047b 100644 --- a/docs/website/docs/concepts/diagnostics.md +++ b/docs/website/docs/concepts/diagnostics.md @@ -74,9 +74,10 @@ data spells that stops the search, with no `endobj` of the object's own before i guard, and so does an `endstream` the data spells — inside a string, say, an `endobj` after it or not —: the data is marked cut by the guard, whose report the document holds, whether the object ends with `endobj` is not judged, and nothing is said of the file. Only what follows the object's end — its `endobj`, or the `endstream` where none follows — -can prove an `endstream` the stream's own: the next object the index places, the end of the file, or the `xref` or -`startxref` keyword, which data that goes on is never followed by. The end is then the file's, and so is an `endobj` -the object lacks. A whole reference the cut left, `6 0 R` before the window's end, is no length +can prove an `endstream` the stream's own: the next object the index places or the end of the file, which data that +goes on is never followed by, or the `xref` or `startxref` keyword, which the data may spell as well — data that spells +`endstream`, then one of them, and goes on is taken to end there, which only the hidden length could tell. The end is +then the file's, and so is an `endobj` the object lacks. A whole reference the cut left, `6 0 R` before the window's end, is no length it hid: it leads on to the object it names, as read whole. A cross-reference stream whose `/Length` the chain cannot read yet, which a valid file may hold, is searched as well, up diff --git a/docs/website/docs/concepts/lazy-reading.md b/docs/website/docs/concepts/lazy-reading.md index eea67434..8f8f48a0 100644 --- a/docs/website/docs/concepts/lazy-reading.md +++ b/docs/website/docs/concepts/lazy-reading.md @@ -58,8 +58,8 @@ can take 436 MB. An object the map does not hold is one the file does not define, and a reference to it reads as null, as the specification says — or one a limit left out of the map, which reads as null too, until the limit is raised: the sections past `MaxXRefSectionCount`, the rows past `MaxXRefSectionLength` or past what `MaxStreamLength` reads or -`MaxDecodedStreamLength` decodes of a cross-reference stream, a cross-reference stream whose `/W`, `/Index`, `/Size`, -`/Filter` or `/DecodeParms` leads to an object `MaxObjectLength` cuts and the sections older than it — so too one whose +`MaxDecodedStreamLength` decodes of a cross-reference stream, a cross-reference stream whose `/W`, `/Index`, `/Filter` +or `/DecodeParms`, or `/Size` with no `/Index`, leads to an object `MaxObjectLength` cuts and the sections older than it — so too one whose `/Length` leads to such an object, when no `endstream` ends its data before the next object or the end of the file, or the rows past an `endstream` its rows spell, which only that length could tell from the end of its data —, the sections a `/Prev` or an `/XRefStm` that leads to such an object names — the older ones, or the cross-reference stream of a hybrid diff --git a/docs/website/docs/concepts/validation.md b/docs/website/docs/concepts/validation.md index 980369ff..7e8dbc79 100644 --- a/docs/website/docs/concepts/validation.md +++ b/docs/website/docs/concepts/validation.md @@ -155,9 +155,10 @@ The document's [reader limits](reader-limits.md) apply to validation too. An obj limit is not a fault of the file, so a rule meeting one judges nothing of what it lacks or holds, or says, as information, that it could not check it whole, rather than calling the file broken; opening the document with a raised limit lets it check the rest. The same holds of a value that leads to such an object: a cross-reference stream whose -`/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` leads to one is said not checked rather than malformed — so is one +`/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, leads to one is said not checked rather than +malformed — so is one whose `/Length`, which the chain could not read, leads to one where only that length could say where its data ends —; -a `/Prev` or an `/XRefStm` that leads to one ends the chain there rather than naming a section not found; and an +a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain there rather than naming a section not found; and an object stream whose `/Filter` or `/DecodeParms` leads to one, or to an object a limit left out of the index, is left encoded, the objects it holds said not checked rather than lost ([#263](https://github.com/AdCodicem/AdCodicem.Pdf/issues/263)). The page tree rules take a node so cut to give what its diff --git a/docs/website/docs/reference/diagnostics.md b/docs/website/docs/reference/diagnostics.md index 5ca4f961..a951933a 100644 --- a/docs/website/docs/reference/diagnostics.md +++ b/docs/website/docs/reference/diagnostics.md @@ -102,7 +102,7 @@ Codes are stable: they are part of the public contract, because callers filter o | `xref.rebuilt` | The index was rebuilt by scanning the whole file | | `xref.offset-adjusted` | An object, or a cross-reference section the chain names, was not where the file said, and was found nearby | | `xref.section-missing` | A cross-reference section `/Prev` or `/XRefStm` names holds nothing that reads as a section, nor does any place nearby, lies outside the file, or is named by a value that is not an offset; what only it indexed is found by rebuilding the index when it is asked for. Reported at the offset the section was named at, or, when that offset lies outside the file or the value is not an offset, at the section whose trailer named it. A `/Prev` or `/XRefStm` written as a reference that leads to an object one of the reader's limits cut names what the guard hides: no section is reported missing, nothing is rebuilt, the chain ends there, and the guard is reported once the chain is read | -| `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, `/Size` or data cannot give rows, or whose `/W`, `/Index`, `/Size`, `/Filter` or `/DecodeParms` is a reference no section read before it places where it can be read without loading it, or whose `/Length` is such a reference, or leads to one through others, and no `endstream` follows its data before the next object the sections read so far place, or before the end of the file, or none was looked for once the document's searches for `endstream` read as much of the file as they may — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message. A value that names an object one of the reader's limits cut, directly or through a chain of references — a `/Length` whose data no `endstream` ends before that stop among them —, is the guard's, not the file's: the section is not reported, nor looked for nearby, the chain ends there, nothing is rebuilt, what only that section and the older ones index reads as null, and the guard is reported once the chain is read | +| `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, data or — with no `/Index` — `/Size` cannot give rows, or whose `/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, is a reference no section read before it places where it can be read without loading it, or whose `/Length` is such a reference, or leads to one through others, and no `endstream` follows its data before the next object the sections read so far place, or before the end of the file, or none was looked for once the document's searches for `endstream` read as much of the file as they may — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message. A value that names an object one of the reader's limits cut, directly or through a chain of references — a `/Length` whose data no `endstream` ends before that stop among them —, is the guard's, not the file's: the section is not reported, nor looked for nearby, the chain ends there, nothing is rebuilt, what only that section and the older ones index reads as null, and the guard is reported once the chain is read | | `xref.chain-cycle` | The chain of previous sections looped: a `/Prev` or `/XRefStm` names an offset the chain has already named, whether it read a section there or not. Reported at that offset, or, when it lies outside the file, at the section whose trailer named it | | `xref.entry-out-of-range` | An object's entry places it outside the file — past its end, before its start, or past what a `long` holds once the header's offset is added —; reported with no position, the offset in the message, exactly. A section named outside the file is `xref.section-missing`'s, or the rebuild's when `startxref` names it | | `stream.length-invalid` | A stream's `/Length` is not where its data ends: its `endstream` lies elsewhere and ends the data; or the `/Length` is no length — absent, not a non-negative integer, or naming an object the file lacks or that could not be read — and the `endstream` ends the data; or no `endstream` lies before the next object or the end of the file, no `endobj` of its object ends the data, and the data is the declared length, when it ends before that stop, or runs up to the next object; or the document's searches for `endstream` have read as much of the file as they may, and the data is taken without a search. A `/Length` that names an object a guard cut, or left out of the index, is not judged, though a missing `endstream` is reported all the same. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case. A cross-reference stream whose `/Length` the chain could not read, its data taken up to its first `endstream`, has it read once the chain is, unless a guard cut that data: a `/Length` that is no length, or that no `endstream` confirms, is reported as any stream's, and one an `endstream` past the one the chain stopped at confirms is reported with the rows the chain left unread. Where the section's own row for that `/Length` lies among the rows the data holds, and gives a length an `endstream` confirms, the data is taken again to it before the rows are read, and nothing is reported; nor is anything of a `/Length` a guard cut or left out of the index | diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index 6dc40685..febe51ff 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -116,8 +116,9 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. the data spells, as an embedded PDF file's does, or at an `endstream` the data spells, as a string may, an `endobj` after it or not: the length that tells those from the end of the data is the guard's. An `endstream` ends data that is the file's only where what follows its object's end — its `endobj`, or the `endstream` itself where none follows — - proves it the stream's own: the next object the index places, the end of the file, or the `xref` or `startxref` - keyword, which no data that goes on can be followed by. Elsewhere the data up to there comes back cut by the guard, + proves it the stream's own: the next object the index places or the end of the file, which no data that goes on is + followed by, or the `xref` or `startxref` keyword — which data may spell too: data that spells `endstream`, then + `xref` or `startxref`, and goes on is taken to end there as the file's, which only the hidden length could tell. Elsewhere the data up to there comes back cut by the guard, `CutByGuard`, whether the object ends with `endobj` is not judged, and nothing is said of the file. A reference the cut left whole, the cut falling after `6 0 R`, hides nothing: it is read through to the object it names, as it is read whole, while the chain is read as afterward. Decoding stops at a filter so named, or @@ -139,7 +140,8 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. where its `/Length` so leads and its data was taken to the first `endstream`, are the guard's too, unless what follows that `endstream` proves it the stream's own, as above: they are then the file's to lack, and the index is rebuilt. An object the chain read through a reference the cut left whole is loaded once the chain is read only for a value it - keeps: not for a place it dropped as no section. + keeps, or for what the offset a section is named at holds, which is read as a load reads it, its reports kept: not + for a place near there it guessed the section was moved to and dropped as none. - A stream read from a document decodes under **that document's** limits, however long after opening you decode it. A stream you build in memory decodes under `PdfReaderLimits.Default`. - A classic trailer that ends within the window its cross-reference table was read through is read whole, whatever From 467e37a419c0c486ab104a354ee77810a8778559 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 10:43:11 +0000 Subject: [PATCH 100/146] test: say why a member's stream keyword at the bound is cut though the data could tell The comment compared the member to an object written in the file whose window ends on stream, which since the file asks the byte past the window reads whole. A member is still cut there, though the byte past lies in the decoded data: that is the edge the debt on members ending at the bound is to ask. Refs #282 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs b/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs index 87978863..82d291fa 100644 --- a/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs @@ -789,9 +789,10 @@ public void A_member_dictionary_whose_stream_keyword_MaxObjectLength_leaves_out_ { // Member 3 is a dictionary of 100 bytes and the stream after it, which an object stream cannot hold. The bound ends // the member's window right after the dictionary, inside "stream", or on its last byte, where it may be a longer - // keyword: what follows the dictionary is unknown, as it is of an object written in the file whose window ends - // there, and the member is cut at the guard, not read as a dictionary in silence. A byte past "stream", the member - // is a stream, and reads as null. + // keyword: what follows the dictionary is unknown, and the member is cut at the guard, not read as a dictionary in + // silence — on the last byte, though the byte past it lies in the decoded data, which #282 is to ask, as an object + // written in the file whose window ends there asks the file. A byte past "stream", the member is a stream, and + // reads as null. const string Member = "<< /A (" + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + ") >>\nstream\nabc\nendstream"; Member.IndexOf(">>", StringComparison.Ordinal).Should().Be(98); var file = new TestPdfBuilder() From fea0cd6932766bda4301b757508c7b251d9d637b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 14:24:31 +0000 Subject: [PATCH 101/146] fix: keep the decoding guard a deferred length's data reached when it is decoded again Data a /Length the chain could not read left to an endstream, which a guard on decoding kept from being taken again, is decoded once more knowing its end hidden, as data a guard cut: ASCIIHex or ASCII85 drops the group its end parts. A filter after it, given less, could then stay within MaxDecodedStreamLength: the guard went unreported, the rows past it were charged to the file, the section was incomplete and the index rebuilt, while ThrowOnLimit threw. The second decoding now keeps the first's verdict and its report, the data cut by the guard, and reports it no second time. Refs #220, #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/architecture.md | 3 +- docs/threat-model.md | 2 +- .../Diagnostics/PdfDiagnostics.cs | 39 +++++++++++ .../IO/Filters/PdfFilterPipeline.cs | 34 +++++++--- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 22 ++++++- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 64 +++++++++++++++++++ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 39 +++++++++++ 7 files changed, 188 insertions(+), 15 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 8fc56936..aa0cb9ba 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -147,7 +147,8 @@ separate types, separate severity scales, and vocabularies that never share a co section is the guard's, not malformed; where the rows spell one and no `endstream` confirms what the cut left of the length, the rows past it are the guard's — unless what follows the `endstream` the data was taken to proves it the stream's own, as in item 4, which leaves the rows the data lacks to the file. Data whose end a - guard hides is decoded to the bytes the file gives, only what a filter says of the end silenced. + guard hides is decoded as data a guard cut, a group its end parts dropped and what a filter says of the end + silenced; a decoding guard the first decoding reached stands. ### 3.2 The write path diff --git a/docs/threat-model.md b/docs/threat-model.md index 178dc250..28448ba5 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -283,7 +283,7 @@ Known gaps: | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). A hybrid file's stream's own `/Prev`, which the chain does not follow, going on through the table's, is not read at all, nor are the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place: neither names a section missing nor ends the chain for a guard, and the rule on trailer values judges them as written, a reference they hold not followed ([#263]). An object the guard cut after a whole reference it holds hides nothing: the chain reads through it, as a load does, and loads it once the chain is read. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s`, `A_table_an_xrefstm_names_in_a_stream_s_place_has_its_own_links_left_unread`, `A_prev_written_as_a_reference_to_a_real_is_judged_through_it_only_where_the_reader_follows_it` | Yes | | What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); a place that declares another type — a CIDFont, whose `/W` gives widths —, or whose own dictionary a guard cut, is a guess dropped, the next place tried | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, to the same bytes — not as data a guard cut, whose last group ASCII85 and ASCIIHex drop, which could take a decoding guard's cut away and charge the rows to the file —: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, stays within the bound: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | diff --git a/src/AdCodicem.Pdf/Diagnostics/PdfDiagnostics.cs b/src/AdCodicem.Pdf/Diagnostics/PdfDiagnostics.cs index 974e0899..afbd8952 100644 --- a/src/AdCodicem.Pdf/Diagnostics/PdfDiagnostics.cs +++ b/src/AdCodicem.Pdf/Diagnostics/PdfDiagnostics.cs @@ -129,6 +129,45 @@ internal void RollBack(PdfDiagnosticsMark mark) } } + /// + /// Drops every entry recorded since but the guards reached, their limit.* reports and + /// their count standing, and those the capacity left out counted as left out still: what a reading met beside a guard + /// it is to make again is met again, and the guard stands. The reports decodings moved in that the mark recorded are + /// held again, with the guards' kept, so that a decoding that meets one again does not report it twice. + /// + internal void RollBackBesideLimits(PdfDiagnosticsMark mark) + { + var kept = mark.Count; + var held = mark.DecodingReports is { } recorded ? new Dictionary(recorded) : null; + + for (var i = mark.Count; i < _entries.Count; i++) + { + var entry = _entries[i]; + + if (!entry.Code.StartsWith("limit.", StringComparison.Ordinal)) + { + continue; + } + + _entries[kept++] = entry; + + if (held is not null && _decodingReports?.ContainsKey(entry) == true) + { + held[entry] = held.GetValueOrDefault(entry) + 1; + } + } + + // Every guard reached since the mark was reported once, kept as an entry or left out by the capacity. + var limitsLeftOut = Math.Max(0, _limits - mark.Limits - (kept - mark.Count)); + _entries.RemoveRange(kept, _entries.Count - kept); + _suppressed = mark.Suppressed + limitsLeftOut; + + if (held is not null) + { + _decodingReports = held; + } + } + /// /// Moves every entry recorded since into , which applies /// its own capacity, and drops them from this instance. diff --git a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs index 080e6b54..aa88a8e0 100644 --- a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs +++ b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs @@ -33,7 +33,7 @@ public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diag /// /// Bounds what each filter's output may be and how many filters run, and says what reaching either does. public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diagnostics, PdfLimitGuard guard) => - Decode(stream, diagnostics, guard, endHidden: false, out _, out _); + Decode(stream, diagnostics, guard, endHidden: false, reachedBefore: false, out _, out _); /// /// Decodes a stream's data under its document's guards, and says whether one of them ended decoding — the most filters @@ -56,7 +56,7 @@ public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diag /// Decoding reaches one of its document's guards, and the document was opened to throw when it does. /// public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diagnostics, out bool leftEncoded, out bool cutByGuard) => - Decode(stream, diagnostics, stream.Data.LimitGuard, endHidden: false, out leftEncoded, out cutByGuard); + Decode(stream, diagnostics, stream.Data.LimitGuard, endHidden: false, reachedBefore: false, out leftEncoded, out cutByGuard); /// /// Decodes a stream's data as does, its end, with @@ -71,8 +71,13 @@ public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diag /// it. The data decodes as data a guard cut does — a last group a filter holds in part, which the hidden end may have /// parted, is dropped, and a predictor's last row undone as far as it goes —, though it is not counted as cut by it: a /// group dropped so counts against all the same, the - /// whole data holding at least as much, so that whether the guard is reached does not depend on where the end is - /// hidden. + /// whole data holding at least as much. A filter after it is given less, and may stay within the bound where the whole + /// data would not: a decoding of data whose end is hidden that a decoding of it reached the bound before keeps that + /// verdict, with . + /// + /// + /// Whether a decoding of the same data stopped at before, + /// and reported it: the data comes back cut by the guard, and no filter reports it again. /// /// As the other overload gives it. /// As the other overload gives it. @@ -80,11 +85,17 @@ public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diag /// Decoding reaches one of its document's guards, and the document was opened to throw when it does. /// public static ReadOnlyMemory Decode( - PdfStream stream, PdfDiagnostics? diagnostics, bool endHidden, out bool leftEncoded, out bool cutByGuard) => - Decode(stream, diagnostics, stream.Data.LimitGuard, endHidden, out leftEncoded, out cutByGuard); + PdfStream stream, PdfDiagnostics? diagnostics, bool endHidden, bool reachedBefore, out bool leftEncoded, out bool cutByGuard) => + Decode(stream, diagnostics, stream.Data.LimitGuard, endHidden, reachedBefore, out leftEncoded, out cutByGuard); private static ReadOnlyMemory Decode( - PdfStream stream, PdfDiagnostics? diagnostics, PdfLimitGuard guard, bool endHidden, out bool leftEncoded, out bool cutByGuard) + PdfStream stream, + PdfDiagnostics? diagnostics, + PdfLimitGuard guard, + bool endHidden, + bool reachedBefore, + out bool leftEncoded, + out bool cutByGuard) { diagnostics ??= guard.DocumentDiagnostics; @@ -108,7 +119,7 @@ private static ReadOnlyMemory Decode( guard.Reach(PdfLimit.StreamLength, diagnostics, guard.DescribeStreamLengthCut(), stream.Data.Position); } - return DecodeChain(stream, chain, data, diagnostics, guard, endHidden, out leftEncoded, out cutByGuard); + return DecodeChain(stream, chain, data, diagnostics, guard, endHidden, reachedBefore, out leftEncoded, out cutByGuard); } var decodings = diagnostics.Decodings; @@ -116,7 +127,7 @@ private static ReadOnlyMemory Decode( try { - return DecodeChain(stream, chain, data, decodings, guard, endHidden, out leftEncoded, out cutByGuard); + return DecodeChain(stream, chain, data, decodings, guard, endHidden, reachedBefore, out leftEncoded, out cutByGuard); } finally { @@ -133,6 +144,7 @@ private static ReadOnlyMemory DecodeChain( PdfDiagnostics? diagnostics, PdfLimitGuard guard, bool endHidden, + bool reachedBefore, out bool leftEncoded, out bool cutByGuard) { @@ -144,7 +156,9 @@ private static ReadOnlyMemory DecodeChain( // reported once, by the guard that made it; data whose end is hidden is no cut. A filter drops a last group such // data holds in part, and counts it against the bound all the same (Ascii85Filter, AsciiHexFilter). var cut = stream.Data.CutByGuard || endHidden; - var limitReported = false; + + // A guard a decoding of the same data reached before was reported then: its verdict stands, whatever this one meets. + var limitReported = reachedBefore; // The filters before the last share one bound for what they pass along together, and the last keeps its own: a // stream of one or two filters decodes as each filter alone would, and a chain however long holds, and fills, no diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 530e802a..8a1cc572 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3910,9 +3910,13 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // Whether where the data ends is the guard's to hide, which the data is decoded knowing. var endHidden = false; + // Whether the decoding before stopped at MaxDecodedStreamLength, which the data decoded again knowing its end is + // hidden keeps reached: the whole data holds at least what that decoding read. + var reachedBefore = false; + while (true) { - var decoded = Filters.PdfFilterPipeline.Decode(xrefStream, _diagnostics, endHidden, out var leftEncoded, out var cut); + var decoded = Filters.PdfFilterPipeline.Decode(xrefStream, _diagnostics, endHidden, reachedBefore, out var leftEncoded, out var cut); // Data the guard on filters left encoded holds none of the section's rows: the section is cut before its first. data = leftEncoded ? default : decoded.Span; @@ -3959,11 +3963,23 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // Data a /Length the chain could not read left to an endstream nothing confirmed, where the length that would say // whether it is the data's own is one a guard cut, or where a guard on decoding kept the data, which lacks rows, // from being taken again to the length: where it ends is the guard's, and a filter run out there says nothing of - // the file. The data is decoded once more knowing it, to the same bytes, whose faults stand (#220, #224). + // the file. The data is decoded once more knowing it, as data a guard cut, a group its end parts dropped, the + // faults in what it read standing (#220, #224). A guard on decoding the first decoding reached stands too, with + // its report: a filter after the one that drops the group, given less, may stay within the bound. if (!endHidden && deferred is not null && !xrefStream.Data.CutByGuard && (lengthCut || (!retaken && !decodedWhole && declared > held))) { - _diagnostics.RollBack(decoding); + reachedBefore = cut; + + if (reachedBefore) + { + _diagnostics.RollBackBesideLimits(decoding); + } + else + { + _diagnostics.RollBack(decoding); + } + endHidden = true; continue; } diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 8f800fe6..9970e66a 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -1135,6 +1135,70 @@ public static byte[] StreamSizedThroughAReferenceOverAnOlderSection(bool olderSt return file.ToArray(); } + /// + /// A file of one revision, whose cross-reference stream 5 indexes objects 0 to 39, the page object 39 last, under + /// /Filter [/ASCIIHexDecode /FlateDecode] and a /Length that names object 4, which only stream 5 indexes. Its + /// Flate data is rows 0 to 38 deflated, then a stored block that holds the page's row, with no checksum; its hex text + /// lacks its last digit, a 0 the page's row ends in, and its >. The rows decode to 280 bytes. Read whole, the + /// file is valid but for the hex text's end and the checksum. + /// + public static byte[] HexFlateRowsLackingTheirLastDigit() + { + using var output = new MemoryStream(); + var offsets = new Dictionary(); + + void Text(string text) => output.Write(Encoding.Latin1.GetBytes(text)); + + void Object(int number, string body) + { + offsets[number] = output.Position; + Text(string.Create(CultureInfo.InvariantCulture, $"{number} 0 obj\n{body}\nendobj\n")); + } + + static byte[] Row(int type, long second, int third) => + [(byte)type, (byte)(second >> 24), (byte)(second >> 16), (byte)(second >> 8), (byte)second, (byte)(third >> 8), (byte)third]; + + Text("%PDF-1.5\n"); + Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + Object(2, "<< /Type /Pages /Kids [39 0 R] /Count 1 >>"); + Object(39, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + var four = output.Position; + var five = four + "4 0 obj\n000000\nendobj\n".Length; + var head = new List(); + + for (var i = 0; i < 39; i++) + { + head.AddRange(i switch + { + 0 => Row(0, 0, 65535), + 1 => Row(1, offsets[1], 0), + 2 => Row(1, offsets[2], 0), + 4 => Row(1, four, 0), + 5 => Row(1, five, 0), + _ => Row(0, 0, 0), + }); + } + + using var deflated = new MemoryStream(); + using var deflate = new DeflateStream(deflated, CompressionLevel.SmallestSize, leaveOpen: true); + deflate.Write([.. head]); + + // Flushed, not closed: the deflate data ends synced, with no final block, and the stored block after it is the last. + deflate.Flush(); + var zlib = new List { 0x78, 0xDA }; + zlib.AddRange(deflated.ToArray()); + var tail = Row(1, offsets[39], 0); + zlib.AddRange([0x01, (byte)tail.Length, 0, (byte)~tail.Length, 0xFF]); + zlib.AddRange(tail); + var hex = Convert.ToHexString([.. zlib])[..^1]; + + Object(4, hex.Length.ToString("D6", CultureInfo.InvariantCulture)); + Text(string.Create( + CultureInfo.InvariantCulture, + $"5 0 obj\n<< /Type /XRef /Size 40 /W [1 4 2] /Filter [/ASCIIHexDecode /FlateDecode] /Root 1 0 R /Length 4 0 R >>\nstream\n{hex}\nendstream\nendobj\nstartxref\n{five}\n%%EOF\n")); + return output.ToArray(); + } + /// /// A file of two classic tables. The first indexes the catalog, the page tree, the page and object 12, the highest /// number; the second, an update, writes the catalog, the page tree and the page again, indexes them with object 4, a diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 65f5edf2..9182d6cd 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -949,6 +949,45 @@ public void An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_len guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries); } + [Theory] + [InlineData(279, false)] + [InlineData(279, true)] + [InlineData(200, false)] + public void A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden(int bound, bool throwOnLimit) + { + // Stream 5's rows decode to 280 bytes, through ASCIIHex then Flate, under a /Length only it places. Under 279, Flate + // stops at the bound, and the data, which lacks the page's row, is decoded again knowing its end hidden: ASCIIHex + // drops the digit its end parts, and Flate, given less, stayed within the bound. The guard went unreported, and the + // page's row was charged to the file — the section incomplete, the index rebuilt (#220, #224). + var file = ChainFiles.HexFlateRowsLackingTheirLastDigit(); + + using (var whole = PdfDocument.Open(file)) + { + whole.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.FilterFailed, PdfDiagnosticCodes.FilterFailed); + whole.GetObject(new PdfObjectId(39)).AsDictionary().IsOfType(PdfName.Page).Should().BeTrue(); + } + + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxDecodedStreamLength = bound }, ThrowOnLimit = throwOnLimit }; + + if (throwOnLimit) + { + var opening = () => PdfDocument.Open(file, options); + opening.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxDecodedStreamLength)); + return; + } + + using var document = PdfDocument.Open(file, options); + var section = document.Reader.Structure.Sections.Should().ContainSingle().Which; + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitDecodedStream); + section.CutByLimit.Should().BeTrue(); + section.Incomplete.Should().BeFalse(); + document.GetObject(new PdfObjectId(39)).Should().BeOfType(); + document.WasRepaired.Should().BeFalse(); + var findings = new PdfValidator().Validate(document).Findings.Select(finding => finding.RuleId); + findings.Should().Contain(PdfValidationRuleIds.XRefCheckedInPart).And.NotContain(PdfValidationRuleIds.XRefSectionMalformed); + } + [Fact] public void The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file() { From a5d1e212eb0ead5cbedceb80577e8723e2483209 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 14:28:25 +0000 Subject: [PATCH 102/146] fix: check a deferred length on data MaxStreamLength cut that holds every row Round 7 checked a cross-reference stream's deferred /Length against data that holds every row a section declares whatever a guard on decoding did, since nothing would take that data again; data MaxStreamLength cut was still left out, though the check reads only the length and where the search found the endstream, which no cut moves. The chain then dropped the report that it took the data short, which the whole read gives. Data a guard cut that lacks rows stays unchecked. Refs #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 2 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 12 +++++------ .../DocumentReaderTests.cs | 20 +++++++++++++++++++ 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index 28448ba5..e832d40f 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -283,7 +283,7 @@ Known gaps: | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). A hybrid file's stream's own `/Prev`, which the chain does not follow, going on through the table's, is not read at all, nor are the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place: neither names a section missing nor ends the chain for a guard, and the rule on trailer values judges them as written, a reference they hold not followed ([#263]). An object the guard cut after a whole reference it holds hides nothing: the chain reads through it, as a load does, and loads it once the chain is read. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s`, `A_table_an_xrefstm_names_in_a_stream_s_place_has_its_own_links_left_unread`, `A_prev_written_as_a_reference_to_a_real_is_judged_through_it_only_where_the_reader_follows_it` | Yes | | What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); a place that declares another type — a CIDFont, whose `/W` gives widths —, or whose own dictionary a guard cut, is a guess dropped, the next place tried | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, stays within the bound: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, stays within the bound: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 8a1cc572..c9588eac 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -4039,12 +4039,12 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect } } - // Data a guard cut — MaxStreamLength as the data was read, or a guard on decoding it that left rows lacking — is the - // guard's, which says so (ADR 34), and is not checked against a length: the cut kept the data from being taken again - // to it. Data that holds every row the section declares, which nothing would take again, is checked whatever a guard - // on decoding did, but not against a length a guard cut, which is no length the file gave. - if (deferred is { } check && - (decodedWhole || (!xrefStream.Data.CutByGuard && !lengthCut && declared <= held))) + // Data a guard cut that lacks rows — MaxStreamLength as the data was read, or a guard on decoding it — is the guard's, + // which says so (ADR 34), and is not checked against a length: the cut kept the data from being taken again to it. + // Data that holds every row the section declares, which nothing would take again, is checked whatever a guard did — + // the check reads the length and where the search found the endstream, which no cut moves —, but not against a + // length a guard cut, which is no length the file gave. + if (deferred is { } check && (decodedWhole || (!lengthCut && declared <= held))) { (_deferredLengths ??= [])[xrefStream.Data.Position] = check; } diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 25f4f89f..f10fc5c3 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -2022,6 +2022,26 @@ public void A_deferred_length_s_data_that_holds_every_row_under_a_guard_on_decod document.Diagnostics.Should().Contain(entry => entry.Code == PdfDiagnosticCodes.LimitDecodedStream); } + [Theory] + [InlineData(39, true)] + [InlineData(29, true)] + [InlineData(28, false)] + public void A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length(int bound, bool everyRow) + { + // As above, MaxStreamLength cutting the data past every row, from 29 bytes on: the run the cut parts is the guard's, + // but the length, checked against where the search found the endstream, which no cut moves, says the chain took the + // data short, as read whole. It was left unchecked, as data a guard cut that lacks rows is, which the cut keeps from + // being taken again (#220). + var file = ChainFiles.UpdateOverRunsSpellingAnEndStream(deferred: true); + using var whole = PdfDocument.Open(file); + using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxStreamLength = bound } }); + var invalid = whole.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.StreamLengthInvalid).Which; + + document.Diagnostics.Should().Equal(everyRow ? [document.Diagnostics[0], invalid] : [document.Diagnostics[0]]); + document.Diagnostics[0].Code.Should().Be(PdfDiagnosticCodes.LimitStreamLength); + document.Reader.Structure.Sections[^1].CutByLimit.Should().BeTrue(); + } + [Fact] public void A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends() { From 9af7b1f6f71a3f0d9869b85b894ae53405d14f4a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 14:29:23 +0000 Subject: [PATCH 103/146] test: hold the ASCII85 dropped group's bound one byte short of the group The test of a group dropped and counted against the bound took a bound with no room for it, and one with room for both its bytes: counting one byte less than the group holds passed both, and the whole suite. A bound with room for one of the two now tells them apart. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- tests/AdCodicem.Pdf.Tests/FilterTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/AdCodicem.Pdf.Tests/FilterTests.cs b/tests/AdCodicem.Pdf.Tests/FilterTests.cs index 1b6415ef..4e1fd2ff 100644 --- a/tests/AdCodicem.Pdf.Tests/FilterTests.cs +++ b/tests/AdCodicem.Pdf.Tests/FilterTests.cs @@ -28,12 +28,14 @@ public class FilterTests [Theory] [InlineData(4, true)] + [InlineData(5, true)] [InlineData(6, false)] public void A_group_ascii85_data_that_ends_where_the_file_s_may_not_parts_is_dropped_and_counted_against_the_bound(int maxLength, bool limited) { // "5sdq," is ABCD, and "77I" the part of the group EF's: data the reader cut, or whose end a guard hides, may end // inside the group, whose bytes the missing digits decide. They are not written, but the whole data holds them, - // and a bound they do not fit within is reached (#224). + // and a bound they do not fit within is reached (#224): with no room for them, with room for one of the two, not + // with room for both. var decoded = Ascii85Filter.Decode("5sdq,77I"u8, out var reached, out _, maxLength, cut: true); decoded.Should().Equal("ABCD"u8.ToArray()); From d24e6ef0f7eb005d8e1ef3b8c166c86fc29da7ad Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 14:35:06 +0000 Subject: [PATCH 104/146] fix: read once where a row that names white space before a header places its object The proof that an endstream is a stream's own reads what follows the object's end once, however often the stream is parsed; the check that a row naming the white space before the next header places the object there read past that offset again on every parse, counted with the searches. A stream the cache let go and read again spent what they may read, until it came back cut by the guard, though it was proven its own at first, and decoded otherwise. That offset is now read once too. The bound's reason, ADR 34, the reference, the concepts and the threat model give the proof's cost as 48 or 64 bytes a stream, read once. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- ...-is-readable-and-the-readers-guards-are.md | 2 +- docs/threat-model.md | 2 +- docs/website/docs/concepts/diagnostics.md | 2 +- docs/website/docs/reference/reader-limits.md | 2 +- .../IO/IPdfStreamDataProvider.cs | 7 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 80 +++++++++++++------ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 44 ++++++++++ 7 files changed, 108 insertions(+), 31 deletions(-) diff --git a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md index 731c29f0..fae99ad8 100644 --- a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md +++ b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md @@ -101,7 +101,7 @@ constant with its reason rewritten: a valid file is read past the parser's windo cross-reference stream whose `/Length` the chain cannot read, by the search for the data of a stream whose `/Length` a guard cut or left out of the index, and by white space and comments after a declared length that run past a first read — about twice the file at most; the proof that an `endstream` found under a hidden `/Length` is the stream's -own reads 48 bytes past each, about the file once more, three times it in all (amended on 2026-10-24). +own reads 48 or 64 bytes past each, once, about the file once more, three times it in all (amended on 2026-10-24). `MaxEndStreamGap`, the four bytes of white space accepted between a stream's data and its `endstream`, is removed: more, or a comment, made a right `/Length` a wrong one, and white space and comments are read on as far as they run, counted with the searches — a comment only once white space diff --git a/docs/threat-model.md b/docs/threat-model.md index e832d40f..88b66bec 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -355,7 +355,7 @@ Known gaps: | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | | A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | -| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 bytes past each, about the file once more; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | +| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, about the file once more; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | | Encoded data of any length: a `/Length` of a few digits claiming gigabytes, one past what an array holds, which would exhaust memory at each read, or data decoding hands back as it is, which `MaxDecodedStreamLength` does not bound | `MaxStreamLength` (*The guards*): what is read of one stream's data at once stops at it, what lies past it is cut by a guard and reported as `limit.stream-length` — once for the stream in its document's diagnostics, and with each decoding that meets the cut in diagnostics a caller passes to `Decode`, as `limit.decoded-stream` is —, and the filters decode what was read as data the reader cut ([#170]) | `ReaderLimitsTests.A_stream_cut_at_MaxStreamLength_decodes_what_lies_within_it_as_data_the_reader_cut_and_is_reported_once`, `Data_decoding_hands_back_as_it_is_is_bounded_by_MaxStreamLength_and_not_by_MaxDecodedStreamLength`, `A_length_past_what_an_array_holds_ends_at_the_guard_and_never_in_an_exhausted_memory`, `Decoding_into_diagnostics_of_one_s_own_reports_the_cut_max_stream_length_made_each_time` | No | diff --git a/docs/website/docs/concepts/diagnostics.md b/docs/website/docs/concepts/diagnostics.md index 6bdb047b..33a0c547 100644 --- a/docs/website/docs/concepts/diagnostics.md +++ b/docs/website/docs/concepts/diagnostics.md @@ -99,7 +99,7 @@ what was read first. A file crafted for it can still make what a stream holds de first. What the searches of one document read together is bounded: none starts once they have read four times the file's length. For a valid file, what they read — its cross-reference streams searched, its streams whose length a guard hid, the white space after a declared length read on — comes to about twice the file at most, and the proofs that an `endstream` -found under a hidden length is the stream's own, 48 bytes past each, to about the file once more; the searches of damaged +found under a hidden length is the stream's own, 48 or 64 bytes past each, once, to about the file once more; the searches of damaged streams read twice the file at most — 0.35 of it at most, all searches together, in the test corpus — unless their stretches overlap: one object's header inside another's dictionary, or a header the search does not take — a regular character glued before its number — that the diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index febe51ff..2e07244c 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -191,7 +191,7 @@ the documents producers write. They are not options: |---|---|---| | Nesting of arrays and dictionaries | 128 levels | Real documents nest a handful of levels; the bound keeps a hostile file from exhausting the stack | | A reference that resolves to another reference | followed 32 times | Producers write the object itself; the bound breaks a chain that loops | -| Searches for a stream's `endstream` | four times the file's length, for a document's searches together | A valid stream's `endstream` follows its declared length: the file is read past the bytes an object is parsed through only for the data of a cross-reference stream whose `/Length` the chain cannot read, for the data of a stream whose `/Length` a guard cut or left out of the index, and for white space and comments after a declared length that run past a first read — about twice the file at most —, and the proof that an `endstream` found under a hidden `/Length` is the stream's own reads 48 bytes past each, about the file once more. Past the bound, a stream whose `endstream` the bytes read with its object do not show is not searched for further, and its report says so | +| Searches for a stream's `endstream` | four times the file's length, for a document's searches together | A valid stream's `endstream` follows its declared length: the file is read past the bytes an object is parsed through only for the data of a cross-reference stream whose `/Length` the chain cannot read, for the data of a stream whose `/Length` a guard cut or left out of the index, and for white space and comments after a declared length that run past a first read — about twice the file at most —, and the proof that an `endstream` found under a hidden `/Length` is the stream's own reads 48 or 64 bytes past each, once, about the file once more. Past the bound, a stream whose `endstream` the bytes read with its object do not show is not searched for further, and its report says so | | Objects loaded while loading another | 64 | An indirect `/Length` leads to an integer, and a stream resolves its filters and the parameters they read before decoding any of its data: no level holds decoded data while the next loads | | Object numbers | up to 2,147,483,647 | An object's number is an `int`, and ISO 32000-1's Annex C advises at most 8,388,607 indirect objects. A number past it is refused where it is read — a header, a reference, a subsection, a row, an object stream's header —, never narrowed to one that would name another object. An object is numbered from 1; 0 names the head of the free list, where a reference, a subsection or a row may give it | | An object stream's members | each read to the next one's start, the last to the end of the decoded data | ISO 32000-1 (7.5.7) has a header list its objects by increasing offset, so that each lies before the next one starts: a pair whose offset is not past the one before it, or whose object would start past the data, ends the header there, as a stray token does, and the objects listed before it are read. A member past `MaxObjectLength` is that limit's | diff --git a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs index 6904a015..8965d5ee 100644 --- a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs +++ b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs @@ -76,9 +76,10 @@ internal interface IPdfStreamDataProvider : IPdfFileEdge /// Determines whether the endstream that ends data at , which a search found, /// is the stream's own beyond what its length could say: what follows its object's end — its endobj, where /// says one follows it, or the endstream itself otherwise —, white space and comments - /// between, is the next object the index places, , the end of the file, or the - /// xref or startxref keyword. Anything else may follow an endstream the data spells, which only - /// the length tells from the stream's own (#224). + /// between, is the next object the index places, — there, or past the white space and + /// comments a row names before its header —, the end of the file, or the xref or startxref keyword. + /// Anything else may follow an endstream the data spells, which only the length tells from the stream's own + /// (#224). /// /// /// Read once for each place, as far as the searches for endstream may read; past that, nothing is proven. diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index c9588eac..c0e0442a 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -144,28 +144,30 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// parser holds, and white space after a declared length is read no further than a first read. /// /// - /// Not a guard a valid file reaches (ADR 34), argued here once for every read it counts. A valid file is read past the - /// parser's window in three ways only: a cross-reference stream whose /Length the chain cannot read (#220) is - /// searched once to its endstream; a stream whose /Length a guard cut or left out of the index (#224) is - /// searched once to its endstream, or to a header its data spells, which leaves the data up to it to the guard; - /// and white space after a declared length that runs past a first read is read on once for each place (#174). The data - /// of different streams lies apart, and so do the runs of white space after them: together those reads cover the file - /// once, and one read past each — about twice the file at most. A fourth adds a little to each stream of the second - /// way: the proof that an endstream found is the stream's own (#224) reads past it to its object's end and what - /// follows, three first reads of bytes, which double only as white space runs on — 48 bytes, - /// about what the smallest such stream spans with its header, so that the proofs add the file once more at most: three - /// times it in all, within the four the searches may read. Every other search is of an invalid stream, whose - /// endstream does not follow its declared length. Each reads from a stream's data to the first object header the - /// bytes hold after it, or to the next object the index as written places, whichever is nearer, and its reads grow - /// from twice what the parser holds of the data - /// (): it reads at most about twice what its object spans, and the searches of - /// objects whose headers follow one another read disjoint stretches — the file twice over at most. Only searches that - /// share a stretch read more: objects that overlap, the header of one inside the dictionary of another, before its - /// data; or a stream whose header the search does not take — a regular character glued before its number, where a - /// rebuild's scan takes the digits after it for one — and the index as written does not place, which only a rebuilt - /// one does. This keeps what they read in proportion to the file, as a window keeps the parsing of their - /// dictionaries. Which streams keep their declared length unsearched past it depends on the order they are read in: - /// crafted files reach it, and no document of the corpus comes near, whose searches read 0.35 of a file at most. + /// Not a guard a valid file reaches (ADR 34), argued here once for every read it counts. A valid file is read past + /// the parser's window in four ways only. Three read the data: a cross-reference stream whose /Length the + /// chain cannot read (#220) is searched once to its endstream; a stream whose /Length a guard cut or + /// left out of the index (#224) is searched once to its endstream, or to a header its data spells, which + /// leaves the data up to it to the guard; and white space after a declared length that runs past a first read is + /// read on once for each place (#174). The data of different streams lies apart, and so do the runs of white space + /// after them: together those reads cover the file once, and one read past each — about twice the file at most. The + /// fourth adds a little to each stream of the second way: the proof that an endstream found is the stream's + /// own (#224) reads past it to its object's end and what follows, three first reads of + /// bytes, and a fourth past a row that names the white space before the next object's header, each once however + /// often the stream is parsed, doubling only as white space runs on — 48 or 64 bytes, about what the smallest such + /// stream spans with its header, so that the proofs add the file once more at most: three times it in all, within + /// the four the searches may read. Every other search is of an invalid stream, whose endstream does not + /// follow its declared length. Each reads from a stream's data to the first object header the bytes hold after it, + /// or to the next object the index as written places, whichever is nearer, and its reads grow from twice what the + /// parser holds of the data (): it reads at most about twice what its object + /// spans, and the searches of objects whose headers follow one another read disjoint stretches — the file twice + /// over at most. Only searches that share a stretch read more: objects that overlap, the header of one inside the + /// dictionary of another, before its data; or a stream whose header the search does not take — a regular character + /// glued before its number, where a rebuild's scan takes the digits after it for one — and the index as written + /// does not place, which only a rebuilt one does. This keeps what they read in proportion to the file, as a window + /// keeps the parsing of their dictionaries. Which streams keep their declared length unsearched past it depends on + /// the order they are read in: crafted files reach it, and no document of the corpus comes near, whose searches + /// read 0.35 of a file at most. /// internal const int EndStreamSearchPasses = 4; @@ -407,6 +409,13 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// private readonly Dictionary<(long DataEnd, EndObjState EndObj), (long Position, bool Boundary)> _endStreamFollowers = []; + /// + /// Where the first significant byte lies past each offset a row names before the place it is weighed against, as + /// reads it: -1 where nothing could be read. Read once, however many times its stream is + /// parsed. + /// + private readonly Dictionary _placedStarts = []; + /// /// Where the data starts of each stream whose length fault a reading the reader kept reported: the stream is not /// reported again when it is parsed again. @@ -1956,8 +1965,29 @@ bool IPdfStreamDataProvider.IsOwnEndStream(long absoluteDataEnd, EndObjState end /// Determines whether the object an index places at starts at : /// there, or past the white space and comments its row names, as the reader reads an object at a row's offset. /// - private bool IsPlacedAt(long position, long? placed) => - placed is { } at && (at == position || (at < position && NextSignificant(at, ProofRead) == position)); + /// + /// What lies past an offset is read once, counted with the searches, however many times its stream is parsed. + /// + private bool IsPlacedAt(long position, long? placed) + { + if (placed is not { } at || at > position) + { + return false; + } + + if (at == position) + { + return true; + } + + if (!_placedStarts.TryGetValue(at, out var start)) + { + start = NextSignificant(at, ProofRead); + _placedStarts[at] = start; + } + + return start == position; + } /// /// Determines whether , whole, starts at : the file ends after it, @@ -2227,6 +2257,8 @@ internal void Close() _endStreamChecks.TrimExcess(); _endStreamFollowers.Clear(); _endStreamFollowers.TrimExcess(); + _placedStarts.Clear(); + _placedStarts.TrimExcess(); Release(_lengthFaultsReported); Release(_syntaxReported); Release(_limitsReached); diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 9182d6cd..4e7c3455 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -988,6 +988,50 @@ public void A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decod findings.Should().Contain(PdfValidationRuleIds.XRefCheckedInPart).And.NotContain(PdfValidationRuleIds.XRefSectionMalformed); } + [Fact] + public void The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed() + { + // Stream 10's /Length, object 9, is cut by MaxObjectLength, and object 11, which follows it, is placed by a row + // naming the line feed before its header: the proof reads past that offset to the header. It was read again on + // every parse of the stream, the cache having let it go, counted with the searches each time, until they could read + // no more and the stream, proven its own at first, came back cut by the guard (#224). + var file = new HandWrittenFile(version: "1.5"); + var rows = new Dictionary + { + [1] = (1, file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), 0), + [2] = (1, file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), 0), + [3] = (1, file.Object(3, Page), 0), + [9] = (1, file.Object(9, new string(' ', 2000) + "1"), 0), + [10] = (1, file.Object(10, "<< /Length 9 0 R >>\nstream\nq\nendstream"), 0), + }; + rows[11] = (1, file.Object(11, "(eleven)") - 1, 0); + + for (var number = 12; number < 90; number++) + { + rows[number] = (1, file.Object(number, Invariant($"({number})")), 0); + } + + file.CrossReferenceStream(90, 91, rows, "/Root 1 0 R"); + var options = new PdfReaderOptions { ObjectCacheCapacity = 64, Limits = PdfReaderLimits.Default with { MaxObjectLength = 1024 } }; + using var document = PdfDocument.Open(file.Bytes, options); + + document.GetObject(new PdfObjectId(10)).AsStream().Required().Data.CutByGuard.Should().BeFalse(); + var read = document.Reader.EndStreamSearchBytes; + read.Should().Be(64, "four first reads of 16 bytes: past the data, the endstream, the endobj, and the row's offset"); + + for (var round = 0; round < 20; round++) + { + for (var number = 12; number < 90; number++) + { + _ = document.GetObject(new PdfObjectId(number)); + } + + document.GetObject(new PdfObjectId(10)).AsStream().Required().Data.CutByGuard.Should().BeFalse(); + } + + document.Reader.EndStreamSearchBytes.Should().Be(read); + } + [Fact] public void The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file() { From d6cb581423babcce2ea6011bdc63e99cb1d5934b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 14:39:30 +0000 Subject: [PATCH 105/146] fix: leave to the guard a stream left unsearched whose length a guard hid Once the document's searches have read as much of the file as they may, a stream whose endstream lies past the bytes held is not searched. Under a /Length a guard cut or left out of the index, its data then ran to the next object, the stream's own endstream taken in, and was reported as the file's fault, while an endstream found but not proven is the guard's. Where it ends is unknown, and the length that would say is the guard's: the data up to the stop is cut by it, its endobj unjudged, nothing reported. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 2 +- docs/website/docs/concepts/diagnostics.md | 3 +- docs/website/docs/reference/diagnostics.md | 2 +- docs/website/docs/reference/reader-limits.md | 2 +- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 9 ++++++ .../AdCodicem.Pdf.Tests/StreamLengthTests.cs | 29 +++++++++++++++++++ 6 files changed, 43 insertions(+), 4 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index 88b66bec..de6d6dfe 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -355,7 +355,7 @@ Known gaps: | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | | A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | -| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, about the file once more; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | +| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, about the file once more; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | | Encoded data of any length: a `/Length` of a few digits claiming gigabytes, one past what an array holds, which would exhaust memory at each read, or data decoding hands back as it is, which `MaxDecodedStreamLength` does not bound | `MaxStreamLength` (*The guards*): what is read of one stream's data at once stops at it, what lies past it is cut by a guard and reported as `limit.stream-length` — once for the stream in its document's diagnostics, and with each decoding that meets the cut in diagnostics a caller passes to `Decode`, as `limit.decoded-stream` is —, and the filters decode what was read as data the reader cut ([#170]) | `ReaderLimitsTests.A_stream_cut_at_MaxStreamLength_decodes_what_lies_within_it_as_data_the_reader_cut_and_is_reported_once`, `Data_decoding_hands_back_as_it_is_is_bounded_by_MaxStreamLength_and_not_by_MaxDecodedStreamLength`, `A_length_past_what_an_array_holds_ends_at_the_guard_and_never_in_an_exhausted_memory`, `Decoding_into_diagnostics_of_one_s_own_reports_the_cut_max_stream_length_made_each_time` | No | diff --git a/docs/website/docs/concepts/diagnostics.md b/docs/website/docs/concepts/diagnostics.md index 33a0c547..bb6ad9c6 100644 --- a/docs/website/docs/concepts/diagnostics.md +++ b/docs/website/docs/concepts/diagnostics.md @@ -105,7 +105,8 @@ of it at most, all searches together, in the test corpus — unless their stretc another's dictionary, or a header the search does not take — a regular character glued before its number — that the index as the file wrote it does not place. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object is kept, the data otherwise taken up to it or to the end of the file, and the -report says so. +report says so — but for a stream whose length a guard hid, whose data up to there is the guard's, as an `endstream` +nothing proves is. ## Damaged stream data diff --git a/docs/website/docs/reference/diagnostics.md b/docs/website/docs/reference/diagnostics.md index a951933a..2dcbbcb4 100644 --- a/docs/website/docs/reference/diagnostics.md +++ b/docs/website/docs/reference/diagnostics.md @@ -105,7 +105,7 @@ Codes are stable: they are part of the public contract, because callers filter o | `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, data or — with no `/Index` — `/Size` cannot give rows, or whose `/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, is a reference no section read before it places where it can be read without loading it, or whose `/Length` is such a reference, or leads to one through others, and no `endstream` follows its data before the next object the sections read so far place, or before the end of the file, or none was looked for once the document's searches for `endstream` read as much of the file as they may — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message. A value that names an object one of the reader's limits cut, directly or through a chain of references — a `/Length` whose data no `endstream` ends before that stop among them —, is the guard's, not the file's: the section is not reported, nor looked for nearby, the chain ends there, nothing is rebuilt, what only that section and the older ones index reads as null, and the guard is reported once the chain is read | | `xref.chain-cycle` | The chain of previous sections looped: a `/Prev` or `/XRefStm` names an offset the chain has already named, whether it read a section there or not. Reported at that offset, or, when it lies outside the file, at the section whose trailer named it | | `xref.entry-out-of-range` | An object's entry places it outside the file — past its end, before its start, or past what a `long` holds once the header's offset is added —; reported with no position, the offset in the message, exactly. A section named outside the file is `xref.section-missing`'s, or the rebuild's when `startxref` names it | -| `stream.length-invalid` | A stream's `/Length` is not where its data ends: its `endstream` lies elsewhere and ends the data; or the `/Length` is no length — absent, not a non-negative integer, or naming an object the file lacks or that could not be read — and the `endstream` ends the data; or no `endstream` lies before the next object or the end of the file, no `endobj` of its object ends the data, and the data is the declared length, when it ends before that stop, or runs up to the next object; or the document's searches for `endstream` have read as much of the file as they may, and the data is taken without a search. A `/Length` that names an object a guard cut, or left out of the index, is not judged, though a missing `endstream` is reported all the same. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case. A cross-reference stream whose `/Length` the chain could not read, its data taken up to its first `endstream`, has it read once the chain is, unless a guard cut that data: a `/Length` that is no length, or that no `endstream` confirms, is reported as any stream's, and one an `endstream` past the one the chain stopped at confirms is reported with the rows the chain left unread. Where the section's own row for that `/Length` lies among the rows the data holds, and gives a length an `endstream` confirms, the data is taken again to it before the rows are read, and nothing is reported; nor is anything of a `/Length` a guard cut or left out of the index | +| `stream.length-invalid` | A stream's `/Length` is not where its data ends: its `endstream` lies elsewhere and ends the data; or the `/Length` is no length — absent, not a non-negative integer, or naming an object the file lacks or that could not be read — and the `endstream` ends the data; or no `endstream` lies before the next object or the end of the file, no `endobj` of its object ends the data, and the data is the declared length, when it ends before that stop, or runs up to the next object; or the document's searches for `endstream` have read as much of the file as they may, and the data is taken without a search — but where a guard hid the `/Length`, whose data is then the guard's. A `/Length` that names an object a guard cut, or left out of the index, is not judged, though a missing `endstream` is reported all the same. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case. A cross-reference stream whose `/Length` the chain could not read, its data taken up to its first `endstream`, has it read once the chain is, unless a guard cut that data: a `/Length` that is no length, or that no `endstream` confirms, is reported as any stream's, and one an `endstream` past the one the chain stopped at confirms is reported with the rows the chain left unread. Where the section's own row for that `/Length` lies among the rows the data holds, and gives a length an `endstream` confirms, the data is taken again to it before the rows are read, and nothing is reported; nor is anything of a `/Length` a guard cut or left out of the index | | `stream.truncated` | A stream has no `endstream` before the next object or the end of the file, and its object's `endobj` ends its data — following the declared length, which is kept, or ending the object before that stop —; or neither, no declared length ends before the end of the file, and its data runs to it. Data taken without a search, the document's searches having read as much of the file as they may, is `stream.length-invalid`'s, to the end of the file too. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case | | `stream.self-reference` | An object stream's dictionary names an object the stream holds — as its `/Type`, `/Length`, `/N`, `/First`, a filter or a parameter —: that object reads as null while the stream is decoded, and the stream is decoded without it | | `object-stream.member-moved` | An object is in the object stream its entry names, at another index than the entry gives, and was read where the stream's header lists it. Reported where the stream's data starts, the object, the stream and both indexes in the message | diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index 2e07244c..de8ba180 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -191,7 +191,7 @@ the documents producers write. They are not options: |---|---|---| | Nesting of arrays and dictionaries | 128 levels | Real documents nest a handful of levels; the bound keeps a hostile file from exhausting the stack | | A reference that resolves to another reference | followed 32 times | Producers write the object itself; the bound breaks a chain that loops | -| Searches for a stream's `endstream` | four times the file's length, for a document's searches together | A valid stream's `endstream` follows its declared length: the file is read past the bytes an object is parsed through only for the data of a cross-reference stream whose `/Length` the chain cannot read, for the data of a stream whose `/Length` a guard cut or left out of the index, and for white space and comments after a declared length that run past a first read — about twice the file at most —, and the proof that an `endstream` found under a hidden `/Length` is the stream's own reads 48 or 64 bytes past each, once, about the file once more. Past the bound, a stream whose `endstream` the bytes read with its object do not show is not searched for further, and its report says so | +| Searches for a stream's `endstream` | four times the file's length, for a document's searches together | A valid stream's `endstream` follows its declared length: the file is read past the bytes an object is parsed through only for the data of a cross-reference stream whose `/Length` the chain cannot read, for the data of a stream whose `/Length` a guard cut or left out of the index, and for white space and comments after a declared length that run past a first read — about twice the file at most —, and the proof that an `endstream` found under a hidden `/Length` is the stream's own reads 48 or 64 bytes past each, once, about the file once more. Past the bound, a stream whose `endstream` the bytes read with its object do not show is not searched for further, and its report says so — or, where a guard hid its `/Length`, its data up to the next object is the guard's, `CutByGuard`, nothing reported | | Objects loaded while loading another | 64 | An indirect `/Length` leads to an integer, and a stream resolves its filters and the parameters they read before decoding any of its data: no level holds decoded data while the next loads | | Object numbers | up to 2,147,483,647 | An object's number is an `int`, and ISO 32000-1's Annex C advises at most 8,388,607 indirect objects. A number past it is refused where it is read — a header, a reference, a subsection, a row, an object stream's header —, never narrowed to one that would name another object. An object is numbered from 1; 0 names the head of the free list, where a reference, a subsection or a row may give it | | An object stream's members | each read to the next one's start, the last to the end of the decoded data | ISO 32000-1 (7.5.7) has a header list its objects by increasing offset, so that each lies before the next one starts: a pair whose offset is not past the one before it, or whose object would start past the data, ends the header there, as a stray token does, and the objects listed before it are read. A member past `MaxObjectLength` is that limit's | diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index dc1112d6..35c3d30b 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -872,6 +872,15 @@ private PdfStream FinishWithoutEndStream( if (!search.Searched) { // The document's searches read as much of the file as they may: what lies past the bytes held is not looked at. + // Where a /Length a guard cut or left out of the index would say how far the data runs, where it ends is the + // guard's, as where an endstream found is not proven the stream's own: the data up to the stop is cut by it, the + // object's endobj unjudged, and nothing is said of the file (#224). + if (declared.Form is StreamLengthForm.ObjectCut or StreamLengthForm.ObjectUnindexed) + { + _endObjUnknown = true; + return Finish(dictionary, dataStart, upToStop, span.Length, cutByGuard: true); + } + var kept = fits ? length : upToStop; if (KeepLengthFault(Fault(declared, dataStart, kept, null, EndStreamState.NotSearched, null))) diff --git a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs index e95c8c95..041a19ac 100644 --- a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs +++ b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs @@ -2231,6 +2231,35 @@ public void An_endstream_the_data_spells_past_which_the_searches_may_read_no_mor document.Reader.IsEndObjMissing(Streams + 3, out _).Should().BeFalse(); } + [Fact] + public void A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s() + { + // As above, stream 13's data running 3,000 bytes past the bytes its object is parsed with: once the searches have + // read as much as they may, it is not searched past them, and where it ends is unknown — the length that would say + // is the guard's. The data up to the next object, its endstream and endobj taken in, was the file's, reported as a + // length the reader did not search past; it is the guard's, as an endstream found and not proven is (#224). + const int Streams = 10; + var data = "BT " + new string('x', 3000) + " ET"; + var (file, _) = NestedStreams( + Streams, + 100_000, + Invariant($"{Streams + 3} 0 obj\n<< /Length {Streams + 4} 0 R >>\nstream\n{data}\nendstream\nendobj\n"), + Invariant($"{Streams + 4} 0 obj\n{new string(' ', 2000)}{data.Length}\nendobj\n")); + using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = 1024 } }); + + for (var number = 3; number <= Streams + 2; number++) + { + document.GetObject(new PdfObjectId(number)); + } + + var reports = document.Diagnostics.Count; + var stream = document.GetObject(new PdfObjectId(Streams + 3)).AsStream().Required(); + + stream.Data.CutByGuard.Should().BeTrue(); + document.Reader.IsEndObjMissing(Streams + 3, out _).Should().BeFalse(); + document.Diagnostics.Skip(reports).Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + } + [Fact] public void A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds() { From 3b685418de8d84a7013500f1b858895ffaf647ea Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 14:45:35 +0000 Subject: [PATCH 106/146] fix: prove an endstream by the cross-reference stream the chain read after it Under a hidden /Length, an endstream is the stream's own where what follows the object's end is the next object the index places, among others. A cross-reference stream with no row of its own, as PDFBox writes it, is placed by no row: the content stream before it was never proven, and its data, whole, was the guard's. The sections the chain read now place their objects too, for a load and for the chain alike. Rows a newer section superseded still place nothing; the debt names them. Refs #224, #287 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- ...-is-readable-and-the-readers-guards-are.md | 5 ++-- docs/architecture.md | 4 +-- docs/milestones/M02.md | 4 +-- docs/threat-model.md | 5 +++- docs/website/docs/concepts/diagnostics.md | 4 +-- docs/website/docs/concepts/reader-limits.md | 8 +++--- docs/website/docs/reference/diagnostics.md | 4 +-- docs/website/docs/reference/reader-limits.md | 4 +-- .../IO/IPdfStreamDataProvider.cs | 3 ++- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 23 +++++++++++++++- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 27 +++++++++++++++++++ 11 files changed, 73 insertions(+), 18 deletions(-) diff --git a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md index fae99ad8..553d9b21 100644 --- a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md +++ b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md @@ -117,8 +117,9 @@ to an object a guard cut or left out of the index — a stream's `/Length`, `/Fi filter's parameters, a cross-reference stream's `/W`, `/Index` or `/Size`, a trailer's `/Prev` or `/XRefStm` — is the guard's, not the file's: an integer the cut parted, `1` of `15`, is only a place to look for `endstream` at, never a declared length, though a stream that lacks its `endstream` is still the file's fault, unless a header its data spells, -or an `endstream` nothing after it proves the stream's own — the next object the index places, the end of the file, -`xref` or `startxref`, after the object's `endobj` or in its place (amended on 2026-10-24) —, stops the search, the +or an `endstream` nothing after it proves the stream's own — the next object the index places, a cross-reference +stream the chain read, the end of the file, `xref` or `startxref`, after the object's `endobj` or in its place (amended +on 2026-10-24) —, stops the search, the data up to it then the guard's; a whole reference the cut left leads on, as read whole; a chain of filters ends there, its data left encoded; and a cross-reference section whose values lead to an object `MaxObjectLength` cut ends the diff --git a/docs/architecture.md b/docs/architecture.md index aa0cb9ba..586780a7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -115,8 +115,8 @@ separate types, separate severity scales, and vocabularies that never share a co hidden: when a header its data spells — an embedded file's does — stops its search, no `endobj` of its own before it, or an `endstream` it spells ends it, the data up to there is the guard's, cut by it, and nothing is said of the file. An `endstream` is the stream's own, and its end the file's, only where what follows the object's end — its - `endobj`, or the `endstream` where none follows — is the next object the index places, the end of the file, or - `xref` or `startxref`, read once for each place ([#224]). With no + `endobj`, or the `endstream` where none follows — is the next object the index places, a cross-reference stream + the chain read, the end of the file, or `xref` or `startxref`, read once for each place ([#224]). With no `endstream` before the stop, the object's `endobj` ends the data where it follows the declared length as an `endstream` would, or where it is the last one, a whole keyword — no regular character, solidus or percent sign right before it, which would make it part of a token, a name or a comment that diff --git a/docs/milestones/M02.md b/docs/milestones/M02.md index 633f8c6f..5b0b0a18 100644 --- a/docs/milestones/M02.md +++ b/docs/milestones/M02.md @@ -113,8 +113,8 @@ array and dictionary an object, or a trailer, left open, and the object lacks it A stream's `/Length`, `/Filter` or `/DecodeParms`, or a filter's parameter, that leads to an object a guard cut or left out of the index is the guard's, never judged as a value the file gave: the length is not judged, the data is found by its `endstream`, and one missing is still the file's fault, unless a header the data spells, or an `endstream` nothing -after it proves the stream's own — the next object the index places, the end of the file, `xref` or `startxref`, after -the object's `endobj` or in its place —, stops the search, the data up to it then the guard's; the filters end there, +after it proves the stream's own — the next object the index places, a cross-reference stream the chain read, the end +of the file, `xref` or `startxref`, after the object's `endobj` or in its place —, stops the search, the data up to it then the guard's; the filters end there, the rest of them left undone, `HasImageFilter` seeing no image past it, and an object stream so left is hidden as one a guard cut, `xref.checked-in-part`'s rather than `xref.object-stream-broken`'s. A cross-reference stream whose `/W`, `/Index`, diff --git a/docs/threat-model.md b/docs/threat-model.md index de6d6dfe..2d0c60e2 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -354,7 +354,7 @@ Known gaps: | A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | -| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | +| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | | Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, about the file once more; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | @@ -368,6 +368,8 @@ Known gaps: source not held in memory, whenever their data is asked for. What overlapping streams cost is time rather than memory, bounded for each read by `MaxStreamLength` alone; a check for overlapping data is proposed. - [#138] A stream searched for its `endstream` can keep a length that depends on the object read first. +- [#287] Under a `/Length` a guard hid, an `endstream` followed by an object only a superseded row places — an older + definition an update replaced — is not proven the stream's own: its data, whole, is the guard's. - [#53] Encoded data past about 2 GB cannot be represented: a `/Length` past what an array holds is cut at `MaxStreamLength`, whose ceiling is `Array.MaxLength`. - [#48] Data that decodes past about 2 GB cannot be read whole. @@ -751,3 +753,4 @@ The last column also names the questions they leave open, which that milestone s [#277]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/277 [#279]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/279 [#280]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/280 +[#287]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/287 diff --git a/docs/website/docs/concepts/diagnostics.md b/docs/website/docs/concepts/diagnostics.md index bb6ad9c6..d39a1933 100644 --- a/docs/website/docs/concepts/diagnostics.md +++ b/docs/website/docs/concepts/diagnostics.md @@ -74,8 +74,8 @@ data spells that stops the search, with no `endobj` of the object's own before i guard, and so does an `endstream` the data spells — inside a string, say, an `endobj` after it or not —: the data is marked cut by the guard, whose report the document holds, whether the object ends with `endobj` is not judged, and nothing is said of the file. Only what follows the object's end — its `endobj`, or the `endstream` where none follows — -can prove an `endstream` the stream's own: the next object the index places or the end of the file, which data that -goes on is never followed by, or the `xref` or `startxref` keyword, which the data may spell as well — data that spells +can prove an `endstream` the stream's own: the next object the index places, or a cross-reference stream the chain +read, or the end of the file, which data that goes on is never followed by, or the `xref` or `startxref` keyword, which the data may spell as well — data that spells `endstream`, then one of them, and goes on is taken to end there, which only the hidden length could tell. The end is then the file's, and so is an `endobj` the object lacks. A whole reference the cut left, `6 0 R` before the window's end, is no length it hid: it leads on to the object it names, as read whole. diff --git a/docs/website/docs/concepts/reader-limits.md b/docs/website/docs/concepts/reader-limits.md index 685a26fa..8034d355 100644 --- a/docs/website/docs/concepts/reader-limits.md +++ b/docs/website/docs/concepts/reader-limits.md @@ -55,9 +55,11 @@ as a value the file gave; a whole reference it left, though, leads on to the obj of a stream whose `/Length` is so named is found by its `endstream`, and its length is not judged, though an `endstream` the data lacks is still the file's to lack — unless an object header the data spells stops the search, or an `endstream` it spells, which only the hidden length could tell from the end of the data: the data is then the guard's -too. An `endstream` is the stream's own beyond doubt only where what follows its object's end is something data that -goes on cannot be followed by — the next object the index places, the end of the file, a cross-reference section's -keyword —, and the end is then the file's. Decoding stops at a filter so named, or one whose parameters are, and leaves +too. An `endstream` is taken for the stream's own only where what follows its object's end is the next object the +index places, a cross-reference stream the chain read, or the end of the file, which data that goes on is never followed +by, or a cross-reference section's keyword, `xref` or `startxref`, which the data may spell as well: data that spells +`endstream`, then one of them, and goes on is taken to end there, which only the hidden length could tell. The end is +then the file's. Decoding stops at a filter so named, or one whose parameters are, and leaves the rest of the data encoded, as past the most filters the reader runs; and a `/Prev` or an `/XRefStm` the reader follows that is so named, or a cross-reference stream whose values lead to an object a guard cut, ends the chain as a section a guard cut does, wherever near its named offset the section is found. diff --git a/docs/website/docs/reference/diagnostics.md b/docs/website/docs/reference/diagnostics.md index 2dcbbcb4..4bba7191 100644 --- a/docs/website/docs/reference/diagnostics.md +++ b/docs/website/docs/reference/diagnostics.md @@ -189,8 +189,8 @@ header the data spells, as an embedded PDF file's does, that stops the search it —, or an `endstream` the data spells, an `endobj` after it or not, leaves the data up to it to the guard: nothing is reported of the stream, whose data is cut by the guard, and whether its object ends with `endobj` is not judged. An `endstream` is taken for the stream's own, its end the file's, only where what follows its object's end — its -`endobj`, or the `endstream` where none follows — is the next object the index places, the end of the file, or the -`xref` or `startxref` keyword. +`endobj`, or the `endstream` where none follows — is the next object the index places, a cross-reference stream the +chain read, the end of the file, or the `xref` or `startxref` keyword. ```text Warning stream.length-invalid at 5501: The stream declared 19954 bytes but ended after 19952. diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index de8ba180..30ef132c 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -116,8 +116,8 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. the data spells, as an embedded PDF file's does, or at an `endstream` the data spells, as a string may, an `endobj` after it or not: the length that tells those from the end of the data is the guard's. An `endstream` ends data that is the file's only where what follows its object's end — its `endobj`, or the `endstream` itself where none follows — - proves it the stream's own: the next object the index places or the end of the file, which no data that goes on is - followed by, or the `xref` or `startxref` keyword — which data may spell too: data that spells `endstream`, then + proves it the stream's own: the next object the index places, a cross-reference stream the chain read, or the end of + the file, which no data that goes on is followed by, or the `xref` or `startxref` keyword — which data may spell too: data that spells `endstream`, then `xref` or `startxref`, and goes on is taken to end there as the file's, which only the hidden length could tell. Elsewhere the data up to there comes back cut by the guard, `CutByGuard`, whether the object ends with `endobj` is not judged, and nothing is said of the file. A reference the cut left whole, the cut falling after `6 0 R`, hides nothing: it is read through to the diff --git a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs index 8965d5ee..f54b401b 100644 --- a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs +++ b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs @@ -77,7 +77,8 @@ internal interface IPdfStreamDataProvider : IPdfFileEdge /// is the stream's own beyond what its length could say: what follows its object's end — its endobj, where /// says one follows it, or the endstream itself otherwise —, white space and comments /// between, is the next object the index places, — there, or past the white space and - /// comments a row names before its header —, the end of the file, or the xref or startxref keyword. + /// comments a row names before its header —, a cross-reference stream the chain read, the end of the file, or the + /// xref or startxref keyword. /// Anything else may follow an endstream the data spells, which only the length tells from the stream's own /// (#224). /// diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index c0e0442a..6dbc66d3 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -1926,7 +1926,28 @@ private StreamEndSearch SearchEndStreamOnce(int number, long absoluteDataStart, bool IPdfStreamDataProvider.IsOwnEndStream(long absoluteDataEnd, EndObjState endObj, long? nextObject) { var (position, boundary) = EndStreamFollower(absoluteDataEnd, endObj); - return position >= 0 && (boundary || IsPlacedAt(position, nextObject)); + return position >= 0 && (boundary || IsPlacedAt(position, nextObject) || IsStreamSectionAt(position)); + } + + /// + /// Determines whether the object of a cross-reference stream the chain read starts at : the + /// section places it, though no row of its own does — PDFBox writes it so (#224). + /// + /// + /// The sections are as many as lets the chain read, and each says where + /// its object's header lies, past the white space and comments before it: nothing is read. + /// + private bool IsStreamSectionAt(long position) + { + foreach (var section in _structure.Sections) + { + if (section.Kind == XRefSectionKind.Stream && section.StreamObjectNumber > 0 && section.Offset + section.Padding == position) + { + return true; + } + } + + return false; } /// diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 4e7c3455..3250fe3b 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -988,6 +988,33 @@ public void A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decod findings.Should().Contain(PdfValidationRuleIds.XRefCheckedInPart).And.NotContain(PdfValidationRuleIds.XRefSectionMalformed); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public void A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it(bool ownRow) + { + // Content stream 4's /Length, object 5, is cut by MaxObjectLength, and the cross-reference stream follows it, with a + // row of its own or none, as PDFBox writes it: the chain read it there, and it places it all the same. With no row, + // nothing proved the endstream the stream's own, and its data, whole, was the guard's (#224). + var file = new HandWrittenFile(version: "1.5"); + var rows = new Dictionary + { + [1] = (1, file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), 0), + [2] = (1, file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), 0), + [3] = (1, file.Object(3, Page), 0), + [5] = (1, file.Object(5, new string(' ', 2000) + "5"), 0), + [4] = (1, file.Object(4, "<< /Length 5 0 R >>\nstream\nBT ET\nendstream"), 0), + }; + file.CrossReferenceStream(6, ownRow ? 7 : 6, rows, "/Root 1 0 R"); + using var document = PdfDocument.Open(file.Bytes, ObjectGuard); + + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + stream.Data.CutByGuard.Should().BeFalse(); + stream.GetRawBytes().ToArray().Should().Equal("BT ET"u8.ToArray()); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + } + [Fact] public void The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed() { From be4990aa533e573449c5ef19dccaa0a5dcdaa798 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 14:55:28 +0000 Subject: [PATCH 107/146] fix: decode as hidden the end of data a proven endstream ends past white space Under a /Length a guard hid, an endstream that what follows proves the stream's own ends data that is the file's. The proof says which endstream it is, not whether the white space before it is the data's: the search leaves out an end-of-line, a carriage return with it, and keeps white space the data ends in. Flate data whose checksum ends in a carriage return lost it, and was reported as the file's fault, which read whole it is not. Such data stays the file's, its endobj judged, but its end is decoded as one the guard hides: what a filter says of it is not the file's. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/architecture.md | 4 +- docs/threat-model.md | 2 +- docs/website/docs/concepts/diagnostics.md | 4 +- docs/website/docs/concepts/reader-limits.md | 3 +- docs/website/docs/reference/reader-limits.md | 4 +- .../IO/Filters/PdfFilterPipeline.cs | 10 ++- .../IO/IPdfStreamDataProvider.cs | 12 ++- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 53 ++++++++++- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 10 ++- src/AdCodicem.Pdf/Objects/PdfStreamData.cs | 8 ++ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 90 ++++++++++++++++--- 11 files changed, 174 insertions(+), 26 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 586780a7..48d8d922 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -116,7 +116,9 @@ separate types, separate severity scales, and vocabularies that never share a co it, or an `endstream` it spells ends it, the data up to there is the guard's, cut by it, and nothing is said of the file. An `endstream` is the stream's own, and its end the file's, only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places, a cross-reference stream - the chain read, the end of the file, or `xref` or `startxref`, read once for each place ([#224]). With no + the chain read, the end of the file, or `xref` or `startxref`, read once for each place ([#224]); white space + before the keyword, which only the hidden length says is the data's or not, leaves the data's end to the guard as it + decodes (`PdfStreamData.EndHidden`). With no `endstream` before the stop, the object's `endobj` ends the data where it follows the declared length as an `endstream` would, or where it is the last one, a whole keyword — no regular character, solidus or percent sign right before it, which would make it part of a token, a name or a comment that diff --git a/docs/threat-model.md b/docs/threat-model.md index 2d0c60e2..a3e7f2e3 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -354,7 +354,7 @@ Known gaps: | A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | -| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | +| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`), so that a Flate checksum short of a carriage return, or a group a filter holds in part, says nothing of the file. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | | Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, about the file once more; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | diff --git a/docs/website/docs/concepts/diagnostics.md b/docs/website/docs/concepts/diagnostics.md index d39a1933..299b7b24 100644 --- a/docs/website/docs/concepts/diagnostics.md +++ b/docs/website/docs/concepts/diagnostics.md @@ -77,7 +77,9 @@ nothing is said of the file. Only what follows the object's end — its `endobj` can prove an `endstream` the stream's own: the next object the index places, or a cross-reference stream the chain read, or the end of the file, which data that goes on is never followed by, or the `xref` or `startxref` keyword, which the data may spell as well — data that spells `endstream`, then one of them, and goes on is taken to end there, which only the hidden length could tell. The end is -then the file's, and so is an `endobj` the object lacks. A whole reference the cut left, `6 0 R` before the window's end, is no length +then the file's, and so is an `endobj` the object lacks — but for white space before the keyword, an end-of-line +above all, which only the hidden length says is the data's or not: its end is decoded as one the guard hides, and a +filter that runs out there, a Flate checksum short of a carriage return, says nothing of the file. A whole reference the cut left, `6 0 R` before the window's end, is no length it hid: it leads on to the object it names, as read whole. A cross-reference stream whose `/Length` the chain cannot read yet, which a valid file may hold, is searched as well, up diff --git a/docs/website/docs/concepts/reader-limits.md b/docs/website/docs/concepts/reader-limits.md index 8034d355..de3b7b2f 100644 --- a/docs/website/docs/concepts/reader-limits.md +++ b/docs/website/docs/concepts/reader-limits.md @@ -59,7 +59,8 @@ too. An `endstream` is taken for the stream's own only where what follows its ob index places, a cross-reference stream the chain read, or the end of the file, which data that goes on is never followed by, or a cross-reference section's keyword, `xref` or `startxref`, which the data may spell as well: data that spells `endstream`, then one of them, and goes on is taken to end there, which only the hidden length could tell. The end is -then the file's. Decoding stops at a filter so named, or one whose parameters are, and leaves +then the file's, though white space before the keyword, which only the length says is the data's or not, leaves where it +ends to the guard as the data decodes: a filter that runs out there says nothing of the file. Decoding stops at a filter so named, or one whose parameters are, and leaves the rest of the data encoded, as past the most filters the reader runs; and a `/Prev` or an `/XRefStm` the reader follows that is so named, or a cross-reference stream whose values lead to an object a guard cut, ends the chain as a section a guard cut does, wherever near its named offset the section is found. diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index 30ef132c..c495a884 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -118,7 +118,9 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. is the file's only where what follows its object's end — its `endobj`, or the `endstream` itself where none follows — proves it the stream's own: the next object the index places, a cross-reference stream the chain read, or the end of the file, which no data that goes on is followed by, or the `xref` or `startxref` keyword — which data may spell too: data that spells `endstream`, then - `xref` or `startxref`, and goes on is taken to end there as the file's, which only the hidden length could tell. Elsewhere the data up to there comes back cut by the guard, + `xref` or `startxref`, and goes on is taken to end there as the file's, which only the hidden length could tell; white + space before the keyword, which only the length says is the data's or not, leaves the data's end to the guard as it + decodes, so that a filter that runs out there says nothing of the file. Elsewhere the data up to there comes back cut by the guard, `CutByGuard`, whether the object ends with `endobj` is not judged, and nothing is said of the file. A reference the cut left whole, the cut falling after `6 0 R`, hides nothing: it is read through to the object it names, as it is read whole, while the chain is read as afterward. Decoding stops at a filter so named, or diff --git a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs index aa88a8e0..d8e4a782 100644 --- a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs +++ b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs @@ -152,10 +152,12 @@ private static ReadOnlyMemory DecodeChain( // Whether the data a filter is given ends where the file's may not: data the reader cut at one of its limits — // MaxStreamLength, or the window a guard stopped at, cut the stream's data, or a filter before it stopped at - // MaxDecodedStreamLength —, or data whose end a guard hides. Every filter after it is given such data, and a cut is - // reported once, by the guard that made it; data whose end is hidden is no cut. A filter drops a last group such - // data holds in part, and counts it against the bound all the same (Ascii85Filter, AsciiHexFilter). - var cut = stream.Data.CutByGuard || endHidden; + // MaxDecodedStreamLength —, or data whose end a guard hides: the caller says so, or the data does, its endstream + // proven its own past white space only a hidden length says is the data's or not. Every filter after it is given + // such data, and a cut is reported once, by the guard that made it; data whose end is hidden is no cut. A filter + // drops a last group such data holds in part, and counts it against the bound all the same (Ascii85Filter, + // AsciiHexFilter). + var cut = stream.Data.CutByGuard || endHidden || stream.Data.EndHidden; // A guard a decoding of the same data reached before was reported then: its verdict stands, whatever this one meets. var limitReported = reachedBefore; diff --git a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs index f54b401b..25654573 100644 --- a/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs +++ b/src/AdCodicem.Pdf/IO/IPdfStreamDataProvider.cs @@ -18,9 +18,10 @@ internal interface IPdfStreamDataProvider : IPdfFileEdge /// /// Creates stream data for bytes at , marked cut by a guard - /// when : a guard hid what would have said where the data ends (#224). + /// when : a guard hid what would have said where the data ends (#224); or, with + /// , the data's, its end only the guard's to hide. /// - PdfStreamData Create(long absoluteOffset, int length, bool cutByGuard = false); + PdfStreamData Create(long absoluteOffset, int length, bool cutByGuard = false, bool endHidden = false); /// /// Gives where the data of a stream starts in the file: past the end-of-line after its stream keyword, which ends @@ -87,6 +88,13 @@ internal interface IPdfStreamDataProvider : IPdfFileEdge /// bool IsOwnEndStream(long absoluteDataEnd, EndObjState endObj, long? nextObject); + /// + /// Determines whether data a search ended at , at an endstream, may end + /// elsewhere among the white space before the keyword: white space lies between them, which the search left out, or the + /// data's last byte is white space. Only a length says which (#224). + /// + bool EndsBesideWhiteSpace(long absoluteDataEnd); + /// /// Resolves the object a stream's /Length names, and says whether the file holds it and whether it could be /// read — a reference to the stream itself, or to an object inside an object stream being decoded, cannot be —, or diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 6dbc66d3..b2cc54c5 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -416,6 +416,12 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// private readonly Dictionary _placedStarts = []; + /// + /// Where the first significant byte lies past each place a search ended data at, the endstream keyword it found: + /// -1 where nothing could be read. Read once, however many times its stream is parsed. + /// + private readonly Dictionary _endStreamKeywords = []; + /// /// Where the data starts of each stream whose length fault a reading the reader kept reported: the stream is not /// reported again when it is parsed again. @@ -1599,7 +1605,7 @@ private long EntryPosition(XRefEntry entry) long IPdfStreamDataProvider.SourceLength => _source.Length; /// - PdfStreamData IPdfStreamDataProvider.Create(long absoluteOffset, int length, bool cutByGuard) + PdfStreamData IPdfStreamDataProvider.Create(long absoluteOffset, int length, bool cutByGuard, bool endHidden) { // A declared length is a claim made by the file, so it is clamped to what the file can hold. // Without this, "/Length 2147483647" would ask for two gigabytes of memory. @@ -1611,6 +1617,11 @@ PdfStreamData IPdfStreamDataProvider.Create(long absoluteOffset, int length, boo data.MarkCutByGuard(); } + if (endHidden) + { + data.MarkEndHidden(); + } + return data; } @@ -1929,6 +1940,36 @@ bool IPdfStreamDataProvider.IsOwnEndStream(long absoluteDataEnd, EndObjState end return position >= 0 && (boundary || IsPlacedAt(position, nextObject) || IsStreamSectionAt(position)); } + /// + bool IPdfStreamDataProvider.EndsBesideWhiteSpace(long absoluteDataEnd) + { + // The keyword is found as the proof found it, once for each place: past the data's end, white space or comments. + if (absoluteDataEnd <= 0 || absoluteDataEnd > _source.Length) + { + return false; + } + + Span last = stackalloc byte[1]; + return SignificantAfterData(absoluteDataEnd) != absoluteDataEnd || + (_source.Read(absoluteDataEnd - 1, last) == 1 && PdfCharacters.IsWhitespace(last[0])); + } + + /// + /// Gives where the first significant byte lies past , where a search ended data: the + /// endstream keyword it found, past the white space and comments it left out; -1 where nothing could be read. + /// Read once for each place, counted with the searches. + /// + private long SignificantAfterData(long dataEnd) + { + if (!_endStreamKeywords.TryGetValue(dataEnd, out var keyword)) + { + keyword = NextSignificant(dataEnd, ProofRead); + _endStreamKeywords[dataEnd] = keyword; + } + + return keyword; + } + /// /// Determines whether the object of a cross-reference stream the chain read starts at : the /// section places it, though no row of its own does — PDFBox writes it so (#224). @@ -1964,7 +2005,7 @@ private bool IsStreamSectionAt(long position) } // "endstreamxref" is no keyword, and ends no data: what follows it proves nothing. - var keyword = NextSignificant(dataEnd, ProofRead); + var keyword = SignificantAfterData(dataEnd); var end = keyword >= 0 && keyword < _source.Length && IsKeywordAt(keyword, StreamEnds.EndStreamKeyword) ? keyword + StreamEnds.EndStreamKeyword.Length : -1; @@ -2280,6 +2321,8 @@ internal void Close() _endStreamFollowers.TrimExcess(); _placedStarts.Clear(); _placedStarts.TrimExcess(); + _endStreamKeywords.Clear(); + _endStreamKeywords.TrimExcess(); Release(_lengthFaultsReported); Release(_syntaxReported); Release(_limitsReached); @@ -6603,6 +6646,8 @@ private sealed class FileStreamData(PdfFileReader reader, long offset, int lengt { private bool _cutByGuard; + private bool _endHidden; + public override int Length => length; public override long Position => offset; @@ -6611,8 +6656,12 @@ private sealed class FileStreamData(PdfFileReader reader, long offset, int lengt internal override bool CutByGuard => _cutByGuard; + internal override bool EndHidden => _endHidden; + public void MarkCutByGuard() => _cutByGuard = true; + public void MarkEndHidden() => _endHidden = true; + /// The document was disposed. /// /// The data runs past , and the document was opened to throw when it does. diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index 35c3d30b..b4a1a9de 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -847,8 +847,11 @@ private PdfStream ReadStream(PdfDictionary dictionary) return Finish(dictionary, dataStart, found, span.Length, cutByGuard: true); } + // Proven its own, the endstream ends data that is the file's; but white space before it, which the search left out + // or took in, only the hidden length says is the data's or not: its end is the guard's to hide when decoded (#224). _endObjAfterStream = _streamData is null ? null : settled.EndObj; - return Finish(dictionary, dataStart, found, span.Length); + var endHidden = hidden && _streamData is { } proven && proven.EndsBesideWhiteSpace(_baseOffset + dataStart + found); + return Finish(dictionary, dataStart, found, span.Length, endHidden: endHidden); } return FinishWithoutEndStream(dictionary, declared, dataStart, length, settled); @@ -1256,12 +1259,13 @@ private bool KeepLengthFault(StreamLengthFault fault) private readonly void WarnLengthFault(string code, string message, int dataStart) => _diagnostics!.Warn(code, message, _baseOffset + dataStart); - private PdfStream Finish(PdfDictionary dictionary, int dataStart, int length, int bufferLength, bool cutByGuard = false) + private PdfStream Finish( + PdfDictionary dictionary, int dataStart, int length, int bufferLength, bool cutByGuard = false, bool endHidden = false) { length = Math.Max(length, 0); var data = _streamData is not null - ? _streamData.Create(_baseOffset + dataStart, length, cutByGuard) + ? _streamData.Create(_baseOffset + dataStart, length, cutByGuard, endHidden) : PdfStreamData.FromMemory(_memory.Slice(dataStart, Math.Min(length, bufferLength - dataStart))); var afterData = (int)Math.Min((long)dataStart + length, bufferLength); diff --git a/src/AdCodicem.Pdf/Objects/PdfStreamData.cs b/src/AdCodicem.Pdf/Objects/PdfStreamData.cs index 5c0db855..9dc2edb7 100644 --- a/src/AdCodicem.Pdf/Objects/PdfStreamData.cs +++ b/src/AdCodicem.Pdf/Objects/PdfStreamData.cs @@ -31,6 +31,14 @@ public abstract class PdfStreamData /// internal virtual bool CutByGuard => false; + /// + /// Gets a value indicating whether where the data ends is the guard's to hide, though the data is the file's: an + /// endstream proven the stream's own ended it under a /Length a guard hid, with white space before the + /// keyword that only the length could say is the data's or not (#224). Decoding then takes its end as one a guard hides, + /// and says nothing of the file there. + /// + internal virtual bool EndHidden => false; + /// Returns the encoded bytes, as they appear in the file, up to the reader's bound. /// /// Data read from a document held in memory is a view of that memory, which keeps it alive and changes as it does. Data diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 3250fe3b..f40380fd 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -784,16 +784,19 @@ public void An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_ } [Theory] - [InlineData("cut", false)] - [InlineData("left out", false)] - [InlineData("cut", true)] - [InlineData("left out", true)] - public void An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault(string guard, bool flate) + [InlineData("cut", false, true)] + [InlineData("left out", false, true)] + [InlineData("cut", true, true)] + [InlineData("left out", true, true)] + [InlineData("cut", true, false)] + [InlineData("left out", true, false)] + public void An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault(string guard, bool flate, bool endOfLine) { // Object 4's endstream lost its endobj, and object 5, which the table places, follows it directly. No endstream the // data spells can be followed by an object the index places, so the one found is the stream's own, whatever the - // guard hid: what the object lacks is the file's, as read whole, and so is the end of its Flate data, which lost its - // last bytes (#224). + // guard hid: what the object lacks is the file's, as read whole. So is the end of its Flate data, which lost its + // last bytes, where the endstream follows it directly; past an end-of-line, which only the hidden length says is + // the data's or not, its end is the guard's to hide, and the run-out says nothing of the file (#224). var data = "BT /F1 12 Tf 72 712 Td (Hello) Tj ET"; if (flate) @@ -805,7 +808,7 @@ public void An_endstream_the_next_object_follows_where_a_guard_hid_the_length_la zlib.Write(Encoding.ASCII.GetBytes(data)); } - data = Encoding.Latin1.GetString(output.ToArray()[..^6]); + data = Encoding.Latin1.GetString(output.ToArray()[..(endOfLine ? ^6 : ^4)]); } var file = new HandWrittenFile(); @@ -817,7 +820,7 @@ public void An_endstream_the_next_object_follows_where_a_guard_hid_the_length_la [20] = file.Object(20, (guard == "cut" ? new string(' ', 2000) : string.Empty) + data.Length.ToString(CultureInfo.InvariantCulture)), [4] = file.Position, }; - file.Raw($"4 0 obj\n<< /Length 20 0 R{(flate ? " /Filter /FlateDecode" : string.Empty)} >>\nstream\n{data}\nendstream\n"); + file.Raw($"4 0 obj\n<< /Length 20 0 R{(flate ? " /Filter /FlateDecode" : string.Empty)} >>\nstream\n{data}{(endOfLine ? "\n" : string.Empty)}endstream\n"); rows[5] = file.Object(5, "(five)"); file.Table(21, rows, "/Size 21 /Root 1 0 R"); var limits = guard == "cut" @@ -832,12 +835,79 @@ public void An_endstream_the_next_object_follows_where_a_guard_hid_the_length_la stream.Data.CutByGuard.Should().BeFalse(); document.Reader.IsEndObjMissing(4, out _).Should().BeTrue(); document.Diagnostics.Select(entry => entry.Code).Should().Equal( - flate + flate && !endOfLine ? [guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries, PdfDiagnosticCodes.FilterFailed] : [guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries]); + stream.Data.EndHidden.Should().Be(endOfLine); new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.ObjectEndObjMissing).Should().BeTrue(); } + [Theory] + [InlineData("cut", "\n")] + [InlineData("left out", "\n")] + [InlineData("cut", "\r\n")] + public void Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file(string guard, string endOfLine) + { + // Content stream 4's Flate data ends in 0x0D, its checksum's last byte, then an end-of-line and its endstream, which the + // next object proves its own. The search leaves a carriage return out before a line feed, the length that would + // say it is the data's hidden by the guard: the data lacked its last byte, and its Flate data, short of its checksum, + // was reported as the file's fault, which read whole it is not (#224). + byte[] zlib = []; + + for (var i = 0; zlib.Length == 0 || zlib[^1] != 0x0D; i++) + { + using var output = new MemoryStream(); + + using (var stream = new ZLibStream(output, CompressionLevel.Optimal, leaveOpen: true)) + { + stream.Write(Encoding.ASCII.GetBytes(Invariant($"BT /F1 12 Tf ({i}) Tj ET"))); + } + + zlib = output.ToArray(); + } + + var data = Encoding.Latin1.GetString(zlib); + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, Page), + [20] = file.Object(20, (guard == "cut" ? new string(' ', 2000) : string.Empty) + data.Length.ToString(CultureInfo.InvariantCulture)), + [4] = file.Object(4, $"<< /Length 20 0 R /Filter /FlateDecode >>\nstream\n{data}{endOfLine}endstream"), + [5] = file.Object(5, "(five)"), + }; + file.Table(21, rows, "/Size 21 /Root 1 0 R"); + var limits = guard == "cut" + ? PdfReaderLimits.Default with { MaxObjectLength = 1024 } + : PdfReaderLimits.Default with { MaxIndexEntries = 20 }; + + using (var whole = PdfDocument.Open(file.Bytes)) + { + _ = whole.GetObject(new PdfObjectId(4)).AsStream().Required().Decode(); + whole.Diagnostics.Should().BeEmpty(); + } + + using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = limits }); + var content = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + content.Decode().ToArray().Should().Equal(ZLibDecompress(zlib)); + content.Data.CutByGuard.Should().BeFalse(); + content.Data.EndHidden.Should().BeTrue(); + document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal( + guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries); + } + + private static byte[] ZLibDecompress(byte[] zlib) + { + using var input = new MemoryStream(zlib); + using var stream = new ZLibStream(input, CompressionMode.Decompress); + using var output = new MemoryStream(); + stream.CopyTo(output); + return output.ToArray(); + } + [Theory] [InlineData("next object", true)] [InlineData("next object", false)] From 28576734adecc34f2ff8c12127e7f408e7deb3a7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 15:06:52 +0000 Subject: [PATCH 108/146] fix: tell a section a guard cut from what only looks like one, where moved or named A place the chain guessed a section was moved to was kept as the guard's only where a value's cut caused it: a stream that wrote /Type /XRef before MaxTrailerLength cut it, or a table whose rows a guard cut, was dropped, the section said missing and the index rebuilt, where named exactly it is the guard's. A stream with no /Size nor /Index whose /W a guard cut was kept, and the real section lost. Where the chain names a font whose /W precedes the cut, the chain ended at the font. A guessed place is now kept where a value's cut caused it, or where it declared itself a section before the cut; a numbering fault in what a section writes direct is the file's, whatever a value a guard cut; and a guard that cut what declares another type cut no section, relocation going on. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- ...-is-readable-and-the-readers-guards-are.md | 3 +- docs/threat-model.md | 2 +- docs/website/docs/reference/reader-limits.md | 8 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 89 +++++++++-- .../IO/XRef/XRefSectionRecord.cs | 7 + .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 144 ++++++++++++++++++ 6 files changed, 235 insertions(+), 18 deletions(-) diff --git a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md index 553d9b21..ac74c12f 100644 --- a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md +++ b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md @@ -207,7 +207,8 @@ guard: active by default, reported when it is reached, and lifted by an option. cut, or whose `/Length` does where only it could say how far the data runs, a `/Prev` or an `/XRefStm` the reader follows that does (amended on 2026-10-23 and 2026-10-24) —, since an older section, or its trailer, would stand for what the cut left out; and a section a guard cut is where it was named, not looked for near there. One found near where it is named, whose values a guard keeps - from being read, is the guard's there too (amended on 2026-10-24). A rebuild that damage forces on such a chain keeps the entries + from being read, or that declared itself a section — the `xref` keyword, `/Type /XRef` — before a guard cut it, is the + guard's there too (amended on 2026-10-24). A rebuild that damage forces on such a chain keeps the entries the chain read; one the chain cannot do without — a guard that leaves it no section — runs as before. - **Or it throws.** `PdfReaderOptions.ThrowOnLimit`, false by default, turns every guard reached into a `PdfLimitExceededException`, a `PdfException` carrying the code and the name of the property: from diff --git a/docs/threat-model.md b/docs/threat-model.md index a3e7f2e3..23597fa8 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -282,7 +282,7 @@ Known gaps: | A classic table of any length | Its window grows ×4 from 64 KB up to `MaxXRefSectionLength`, and the rows read are kept. A table the guard cuts, or whose rows run past a full index, keeps its trailer, found past the rows left unread by a search a megabyte at a time (`ReadTrailerPastRows`): comments are skipped, and anything else no row holds is the section's fault, reported, the search going on to the `trailer` keyword, so that its `/Encrypt` still refuses the document. The search starts at the start of the line the cut fell in, outside any comment and past the table's own keyword; it reads each megabyte once, every byte of it looked at before the next is read, however many comments or stray bytes it holds, and is not made for a place a section may have been moved to, a guess. The chain ends there, since an older section's rows would stand for the objects the unread rows define. What those rows place reads as null until the guard is raised | `ReaderLimitsTests.A_table_whose_keywords_the_bound_cuts_reports_the_bound`, `A_table_its_length_bound_cuts_keeps_its_trailer_and_the_encryption_it_names`, `A_table_its_length_bound_cuts_ends_the_chain_though_its_trailer_names_an_older_section`, `What_ends_the_rows_a_length_bound_cut_is_judged_as_it_is_where_none_did`, `A_table_a_length_bound_cuts_that_runs_to_the_end_of_the_file_has_no_trailer`, `A_table_its_length_bound_cuts_keeps_the_encryption_its_trailer_names_past_what_no_row_holds`, `A_table_cut_before_its_keyword_ends_keeps_its_trailer`, `Rows_past_a_full_index_are_not_read_and_the_trailer_past_them_is`, `A_cut_in_a_comment_among_the_rows_searches_for_the_trailer_from_the_comment_s_start`, `The_trailer_past_rows_left_unread_is_found_wherever_the_search_s_window_ends`, `A_trailer_past_rows_left_unread_that_its_own_bound_cuts_keeps_what_precedes_the_cut`; `DocumentReaderTests.The_search_for_the_trailer_past_rows_left_unread_reads_the_file_once`; `CrossReferenceChainTests.Reads_an_object_a_length_bound_cut_out_of_its_table_as_null_until_the_guard_is_lifted`, `A_section_found_near_where_prev_names_it_is_not_searched_past_the_rows_a_guard_left_unread`; `WindowEdgeTests.A_classic_table_reads_whole_wherever_its_window_edge_falls_near_its_trailer` | No: seeds stay under 120 KB | | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). A hybrid file's stream's own `/Prev`, which the chain does not follow, going on through the table's, is not read at all, nor are the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place: neither names a section missing nor ends the chain for a guard, and the rule on trailer values judges them as written, a reference they hold not followed ([#263]). An object the guard cut after a whole reference it holds hides nothing: the chain reads through it, as a load does, and loads it once the chain is read. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s`, `A_table_an_xrefstm_names_in_a_stream_s_place_has_its_own_links_left_unread`, `A_prev_written_as_a_reference_to_a_real_is_judged_through_it_only_where_the_reader_follows_it` | Yes | -| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); a place that declares another type — a CIDFont, whose `/W` gives widths —, or whose own dictionary a guard cut, is a guess dropped, the next place tried | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild` | No | +| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); so is one that declared itself a section before a guard cut it — a table's `xref` keyword, `/Type /XRef`. A place that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is a guess dropped, the next place tried; where the chain names a section, a guard that cut what declares another type cut no section, and relocation is tried as read whole. A section that numbers its rows wrongly in what it writes direct — neither `/Size` nor `/Index` — is the file's fault, whatever a value a guard cut hides | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_cross_reference_stream_whose_deferred_length_a_guard_cut_is_found_where_it_was_moved`, `A_cross_reference_stream_a_guard_cut_the_dictionary_of_after_its_type_is_the_section_where_it_was_moved`, `A_classic_table_a_guard_cut_the_rows_of_is_the_section_where_it_was_moved`, `A_stream_with_no_numbering_whose_width_a_guard_cut_is_no_section_the_guard_cut`, `A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild` | No | | A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, stays within the bound: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index c495a884..2cd13f93 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -137,8 +137,12 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. `/Length` does when no `endstream` ends its data, are the guard's, as a section it cut is: the chain ends there, the sections it names are not read — the older ones, or a hybrid file's cross-reference stream —, the index is not rebuilt, and the guard is reported once the chain is read, or thrown by `Open`. So does a section found near the - offset it is named at, which such a value keeps from being read; a place there that declares another type — a CIDFont, - whose `/W` gives widths —, or whose own dictionary a guard cut, is not taken for the section, and the next is tried. The rows a cross-reference stream's data lacks, + offset it is named at, which such a value keeps from being read, or which declared itself one — a table's `xref` + keyword, `/Type /XRef` — before a guard cut it; a place there that declares another type — a CIDFont, whose `/W` gives + widths —, or whose dictionary a guard cut before it said what it is, is not taken for the section, and the next is + tried. A guard that cut, where the chain names a section, what declares another type cut no section; and a section + that numbers its rows wrongly in what it writes direct — neither `/Size` nor `/Index` — is the file's fault, whatever a + value a guard cut hides. The rows a cross-reference stream's data lacks, where its `/Length` so leads and its data was taken to the first `endstream`, are the guard's too, unless what follows that `endstream` proves it the stream's own, as above: they are then the file's to lack, and the index is rebuilt. An object the chain read through a reference the cut left whole is loaded once the chain is read only for a value it diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index b2cc54c5..56e55f15 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3094,10 +3094,11 @@ private void ReportMissingSection(string naming, long offset, long namedFrom) /// places in all, over the whole document: each is read through a window /// of up to 64 KB, the file chooses how many headers lie near a section, and a chain can name a section for /// each of its links. Found, the section's record takes what - /// was read there — a section a value it is read with kept from being read too, leading to an object a guard cut, which - /// is not found to be read, the chain ending there as the guard's —; not found, it keeps what the named offset held. A - /// place is a guess: it must declare a cross-reference stream and no other type, and one a guard cut the dictionary of - /// is dropped. + /// was read there — a section a guard kept from being read too, which is not found to be read, the chain ending there as + /// the guard's —; not found, it keeps what the named offset held. A place is a guess: a table's xref keyword, or + /// an object that declares a cross-reference stream and no other type. A guard that kept it from being read keeps it the + /// section where a value it is read with led to an object the guard cut, or where it declared itself one before the cut + /// — the keyword, or /Type /XRef —; a dictionary cut before it said what it is may be anything, and is dropped. /// private bool TryRelocateXRefSection(long offset, XRefSectionRecord section, out long previous, out long hybrid) { @@ -3138,10 +3139,14 @@ private bool TryRelocateXRefSection(long offset, XRefSectionRecord section, out var readThrough = _chainValuesCut?.Count ?? 0; var state = TryReadXRefSection(candidate, attempt, out previous, out hybrid, candidate: true); - // A section a value it is read with kept from being read, leading to an object a guard cut, is the section all the - // same: found where it was moved, it is where a guard cut the chain, as where it is named (#263). A place a guard - // cut the dictionary of says nothing of what is there, and is dropped as any other is, with what it read through. - if (state == XRefSectionState.Read || (state == XRefSectionState.Malformed && attempt.CutByValue)) + // A section a guard kept from being read is the section all the same, where a value it is read with led to an + // object a guard cut, or where what is there declared itself one before the cut — the xref keyword, /Type /XRef —: + // found where it was moved, it is where a guard cut the chain, as where it is named (#263). A place a guard cut the + // dictionary of before it said what it is may be anything, and is dropped as any other is, with what it read + // through. + if (state == XRefSectionState.Read || + (state == XRefSectionState.Malformed && attempt.CutByLimit && + (attempt.CutByValue || attempt.Kind == XRefSectionKind.Table || attempt.TypedXRef))) { section.RelocateTo(attempt); _diagnostics.Repair( @@ -3785,11 +3790,13 @@ private bool ReadSubsection(ref PdfLexer lexer, int first, int count, XRefSectio /// /// /// - /// An object that declares itself a cross-reference stream — /Type /XRef, or a /W — and cannot be - /// read is a malformed section; any other object there, or none, is no section at all. At a place the section may have - /// been moved to (), nothing such an object's reading met is reported or recorded — its - /// search for an endstream aside, which is kept as every search is —: it was read as a section, through a - /// section's window, and is read as itself when something asks for it. + /// An object that declares itself a cross-reference stream — /Type /XRef, or a /W, beside no other + /// /Type at a place the section may have been moved to — and cannot be read is a malformed section; any + /// other object there, or none, is no section at all, and a guard that cut what declares another type cut no + /// section. What it writes direct that numbers its rows wrongly is the file's fault, whatever a value a guard cut + /// hides. At a place the section may have been moved to (), nothing such an object's + /// reading met is reported or recorded — its search for an endstream aside, which is kept as every search is + /// —: it was read as a section, through a section's window, and is read as itself when something asks for it. /// /// /// ISO 32000-1 (7.5.8.2) makes the values of this dictionary direct, the elements of /W and /Index @@ -3854,7 +3861,11 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect section.Kind = XRefSectionKind.Stream; section.StreamObjectNumber = number; section.TrailerPosition = absolute; - section.CutByLimit |= _guardsReached != guards; + section.TypedXRef = dictionary.IsOfType(PdfName.XRef); + + // A guard that cut what declares another type — a font's /W gives widths — cut no section: what is there is the + // file's to be, as read whole (#263). + section.CutByLimit |= _guardsReached != guards && DeclaresOnlyCrossReferenceStream(value); if (value is not PdfStream xrefStream) { @@ -3890,6 +3901,13 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect if (cutObject is { } past) { + // What the section writes direct that makes it unreadable whatever the guard hid is the file's, as read whole. + if (WrittenNumberingFault(dictionary) is { } numberingWritten) + { + section.Fault = numberingWritten; + return XRefSectionState.Malformed; + } + section.Fault = lengthFault; section.CutByLimit = true; section.CutByValue = true; @@ -3914,6 +3932,13 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect if (unreadCut is { } cut) { + // What the section writes direct that makes it unreadable whatever the guard hid is the file's, as read whole. + if (WrittenNumberingFault(dictionary) is { } numberingWritten) + { + section.Fault = numberingWritten; + return XRefSectionState.Malformed; + } + section.CutByLimit = true; section.CutByValue = true; (_chainValuesCut ??= []).Add(cut); @@ -4363,6 +4388,42 @@ private static bool DeclaresCrossReferenceStream(PdfObject value) => (value is PdfStream stream ? stream.Dictionary : value as PdfDictionary) is { } dictionary && (dictionary.IsOfType(PdfName.XRef) || dictionary.ContainsKey(PdfName.W)); + /// + /// Says what is wrong with how a cross-reference stream numbers its rows, as far as it writes it direct: an + /// /Index or /Size written as a reference, or an /Index element, may lead to what a guard cut, and + /// says nothing then. Null when nothing written direct is wrong. + /// + private static string? WrittenNumberingFault(PdfDictionary dictionary) + { + var written = dictionary.GetRaw(PdfName.Index); + + if (written is PdfReference) + { + return null; + } + + if (written is PdfArray ranges) + { + foreach (var element in ranges) + { + if (element is PdfReference) + { + return null; + } + } + + return NumberingFault(written, written, null, out _); + } + + if (written is not (null or PdfNull)) + { + return NumberingFault(written, written, null, out _); + } + + var size = dictionary.GetRaw(PdfName.Size); + return size is PdfReference ? null : NumberingFault(null, null, size?.AsInteger(), out _); + } + /// /// Determines whether what a place a section may have been moved to holds declares itself a cross-reference stream, and /// no other type: /Type /XRef, or a /W beside no /Type that names another. A guess is no section diff --git a/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs b/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs index ff16fed7..6ca20f92 100644 --- a/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs +++ b/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs @@ -80,6 +80,12 @@ public XRefSectionRecord(string namedBy, long namedOffset, long namedFrom) /// public bool CutByValue { get; set; } + /// + /// Gets a value indicating whether the cross-reference stream's dictionary, as far as it was read, gives + /// /Type /XRef: what is there declared itself a section before any guard's cut. + /// + public bool TypedXRef { get; set; } + /// /// Gets a value indicating whether cut the section's /// trailer: what lay past the cut — an older section's /Prev, a key a newer trailer gives — is unknown. @@ -144,6 +150,7 @@ public void RelocateTo(XRefSectionRecord attempt) Incomplete = attempt.Incomplete; CutByLimit = attempt.CutByLimit; CutByValue = attempt.CutByValue; + TypedXRef = attempt.TypedXRef; TrailerCutByLimit = attempt.TrailerCutByLimit; ChainHiddenByGuard = attempt.ChainHiddenByGuard; ChainHiddenAtStream = attempt.ChainHiddenAtStream; diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index f40380fd..265cedfd 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -2168,6 +2168,150 @@ public void A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_wa findings.Should().Contain(PdfValidationRuleIds.XRefSectionShifted); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public void A_cross_reference_stream_whose_deferred_length_a_guard_cut_is_found_where_it_was_moved(bool typed) + { + // The newer table's /Prev lands 3 bytes into stream 7, whose /Length names object 5, which MaxObjectLength cuts before + // its value, and whose endstream is misspelled: no endstream ends its data, and where it would end is the guard's. + // Found 3 bytes away, it is the section moved, which the guard ends the chain at, as where the /Prev names it, whether + // or not it gives its /Type (#263). + var built = ChainFiles.UnderAnUpdate( + (typed ? "/Type /XRef " : string.Empty) + "/Size 8 /W [1 4 2] /Length 5 0 R", new string(' ', 1100) + "56", compressed: false, predicted: false, placed: true); + var text = Encoding.Latin1.GetString(built); + var stream = text.IndexOf("7 0 obj", StringComparison.Ordinal); + var endStream = text.IndexOf("endstream", stream, StringComparison.Ordinal); + text = text[..endStream] + "endstreaX" + text[(endStream + "endstream".Length)..]; + var file = Encoding.Latin1.GetBytes(text.Replace(Invariant($"/Prev {stream} "), Invariant($"/Prev {stream + 3} "), StringComparison.Ordinal)); + + using (var whole = PdfDocument.Open(file)) + { + whole.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.StreamLengthInvalid, PdfDiagnosticCodes.XRefOffsetAdjusted); + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.XRefOffsetAdjusted, PdfDiagnosticCodes.LimitObject); + document.WasRepaired.Should().BeFalse(); + var moved = document.Reader.Structure.Sections[^1]; + moved.Offset.Should().Be(stream); + moved.CutByLimit.Should().BeTrue(); + var findings = new PdfValidator().Validate(document).Findings.Select(finding => finding.RuleId); + findings.Should().Contain([PdfValidationRuleIds.XRefCheckedInPart, PdfValidationRuleIds.XRefSectionShifted]) + .And.NotContain(PdfValidationRuleIds.XRefSectionNotFound); + + var opening = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }).Dispose(); + opening.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); + } + + [Theory] + [InlineData(0)] + [InlineData(3)] + public void A_cross_reference_stream_a_guard_cut_the_dictionary_of_after_its_type_is_the_section_where_it_was_moved(int shift) + { + // Stream 7 writes /Type /XRef, then a string MaxTrailerLength cuts: the guard keeps it from being read. Named 3 bytes + // off, it was dropped as a place that says nothing of what it is — the section said missing, the index rebuilt, an + // error found —, though it declared itself a section before the cut, as named exactly, where it is the guard's (#263). + var built = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] /Pad (" + new string('x', 300) + ") /Length {length}", "0", compressed: false, predicted: false, placed: true); + var text = Encoding.Latin1.GetString(built); + var stream = text.IndexOf("7 0 obj", StringComparison.Ordinal); + var file = Encoding.Latin1.GetBytes(text.Replace(Invariant($"/Prev {stream} "), Invariant($"/Prev {stream + shift} "), StringComparison.Ordinal)); + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxTrailerLength = 128 } }; + + using var document = PdfDocument.Open(file, options); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal( + shift == 0 ? [PdfDiagnosticCodes.LimitTrailer] : [PdfDiagnosticCodes.LimitTrailer, PdfDiagnosticCodes.XRefOffsetAdjusted]); + document.WasRepaired.Should().BeFalse(); + document.Reader.Structure.Sections[^1].CutByLimit.Should().BeTrue(); + var findings = new PdfValidator().Validate(document).Findings.Select(finding => finding.RuleId); + findings.Should().Contain(PdfValidationRuleIds.XRefCheckedInPart).And.NotContain(PdfValidationRuleIds.XRefSectionNotFound); + } + + [Theory] + [InlineData(0)] + [InlineData(2)] + public void A_classic_table_a_guard_cut_the_rows_of_is_the_section_where_it_was_moved(int shift) + { + // The older table's first row lies past MaxXRefSectionLength, beyond a long comment: the guard keeps it from being + // read. Named 2 bytes off, it was dropped — the section said missing, an error found — though its xref keyword + // declares a section, as named exactly, where it is the guard's (#263). + var file = new HandWrittenFile(); + var one = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + var two = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + var three = file.Object(3, Page); + var older = file.Position; + file.Raw(Invariant( + $"xref\n%{new string('x', 2000)}\n0 4\n0000000000 65535 f\r\n{one:D10} 00000 n\r\n{two:D10} 00000 n\r\n{three:D10} 00000 n\r\ntrailer\n<< /Size 4 /Root 1 0 R >>\n")); + var newer = file.Position; + file.Raw(Invariant($"xref\ntrailer\n<< /Size 4 /Root 1 0 R /Prev {older + shift} >>\nstartxref\n{newer}\n%%EOF\n")); + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxXRefSectionLength = 1024 } }; + + using var document = PdfDocument.Open(file.Bytes, options); + + document.Diagnostics.Select(entry => entry.Code).Should().Contain(PdfDiagnosticCodes.LimitXRefSectionLength) + .And.NotContain(PdfDiagnosticCodes.XRefSectionMissing); + var findings = new PdfValidator().Validate(document).Findings.Select(finding => finding.RuleId); + findings.Should().Contain(PdfValidationRuleIds.XRefCheckedInPart).And.NotContain(PdfValidationRuleIds.XRefSectionNotFound); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void A_stream_with_no_numbering_whose_width_a_guard_cut_is_no_section_the_guard_cut(bool named) + { + // Object 6, a stream 61 bytes before stream 7, writes its /W as 5 0 R, which MaxObjectLength cuts, and neither /Size + // nor /Index: no section, whatever the guard hid, as read whole. Where the /Prev names it, or a place inside it, it + // was taken for the section the guard cut, and stream 7, the section, was lost (#263). + var built = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] /Length {length}", + "[1 4 2]" + new string(' ', 1100), + compressed: false, + predicted: false, + placed: true, + six: "<< /W 5 0 R /Length 3 >>\nstream\nabc\nendstream"); + var text = Encoding.Latin1.GetString(built); + var stream = text.IndexOf("\n7 0 obj", StringComparison.Ordinal) + 1; + var six = text.IndexOf("\n6 0 obj", StringComparison.Ordinal) + 1; + var prev = named ? six : six + 21; + var file = Encoding.Latin1.GetBytes(text.Replace(Invariant($"/Prev {stream} "), Invariant($"/Prev {prev} "), StringComparison.Ordinal)); + + using var whole = PdfDocument.Open(file); + using var document = PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); + + document.Diagnostics.Select(entry => entry.Message).Should().Equal(whole.Diagnostics.Select(entry => entry.Message)); + document.Reader.Structure.Sections[^1].Offset.Should().Be(stream); + document.GetObject(new PdfObjectId(3)).AsDictionary().IsOfType(PdfName.Page).Should().BeTrue(); + } + + [Fact] + public void A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut() + { + // The /Prev names object 6 exactly, a CIDFont whose /W precedes what MaxTrailerLength cuts: the guard reached, which + // the place the chain names keeps, cut no section, the font saying what it is. It was taken for the section the + // guard cut, the chain ended there, and the catalog was lost; read whole, the section is found farther on (#263). + var built = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] /Length {length}", + "0", + compressed: false, + predicted: false, + placed: true, + six: "<< /Type /Font /Subtype /CIDFontType2 /W [0 [500]] /BaseFont /" + new string('F', 200) + " >>"); + var text = Encoding.Latin1.GetString(built); + var stream = text.IndexOf("\n7 0 obj", StringComparison.Ordinal) + 1; + var six = text.IndexOf("\n6 0 obj", StringComparison.Ordinal) + 1; + var file = Encoding.Latin1.GetBytes(text.Replace(Invariant($"/Prev {stream} "), Invariant($"/Prev {six} "), StringComparison.Ordinal)); + + using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxTrailerLength = 128 } }); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitTrailer, PdfDiagnosticCodes.XRefOffsetAdjusted); + document.Reader.Structure.Sections[^1].Offset.Should().Be(stream); + document.GetObject(new PdfObjectId(1)).AsDictionary().IsOfType(PdfName.Catalog).Should().BeTrue(); + document.WasRepaired.Should().BeFalse(); + } + [Fact] public void A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild() { From d350aec0b5f48850dbe9a3118177beb298369606 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 15:08:36 +0000 Subject: [PATCH 109/146] fix: make room in the index by a newest /Size written as a real with no fractional part Round 7 made room by a /Size written direct, but took only an integer: a direct real such as 8.0, which the reader reads as the integer it is everywhere else, no longer bounded the room made, and the map doubled past what the newest /Size numbers. A reference is still not read through for it. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 2 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 7 ++++--- tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs | 14 ++++++++++---- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index 23597fa8..9f72867e 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -286,7 +286,7 @@ Known gaps: | A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, stays within the bound: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | -| A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | +| A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, written direct — an integer, or a real with no fractional part; one written as a reference is not read through for it, and bounds nothing —, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | | A rebuild run again and again | At most one per document, and none once a validation's first rule has begun: before it, the validator has it run when a later read still could, so that no read of a rule's rebuilds the index halfway through ([#237]). A missing object rebuilds only when the index is known to be incomplete through damage: what a guard left out of it — a section past `MaxXRefSectionCount`, rows past `MaxXRefSectionLength`, `MaxStreamLength` or `MaxDecodedStreamLength`, a section whose rows need a value `MaxObjectLength` cut ([#263]), numbers past `MaxIndexEntries` — reads as null, unknown to the validator rather than missing, and each later lookup of it throws under `ThrowOnLimit`. A rebuild damage forces on a chain a guard cut keeps the chain's entries the scan does not find. The chain ends wherever the index fills, and wherever a stream's rows are cut, a hybrid file's stream included, so that no older section stands for what the cut rows define; a cut is known by the count of guards reached, which full diagnostics do not drop, not by its report. A catalog or an object stream a guard hid, and the objects such a stream holds, are no reason to rebuild; the validator takes an object stream the chain's index lacks as unknown once a guard stopped the chain, though a rebuild found the stream since | `DocumentReaderTests.A_reference_the_chain_cannot_read_rebuilds_nothing_while_the_chain_is_read`; `CrossReferenceChainTests.Reads_a_reference_to_an_object_the_file_does_not_define_as_null_without_rebuilding`; `ReaderLimitsTests.A_lookup_past_a_full_index_reads_null_and_rebuilds_nothing`, `A_catalog_past_a_guard_cut_reads_as_null_and_rebuilds_nothing`, `A_rebuild_damage_sets_off_keeps_the_entries_a_chain_a_guard_cut_gave_that_the_scan_cannot_find`; `ReadOrderTests.An_entry_whose_object_a_read_would_not_find_has_the_index_rebuilt_before_the_first_rule`; `CorpusValidationTests.No_read_rebuilds_the_index_after_a_validation`; `CrossReferenceRuleTests.An_object_a_guard_left_out_of_the_index_is_no_reference_the_file_lacks`, `An_index_the_entry_bound_filled_is_said_once_to_be_checked_in_part`; `CrossReferenceChainTests.A_full_index_ends_the_chain_at_the_stream_of_a_hybrid_section`, `An_older_section_does_not_stand_for_what_a_cross_reference_stream_s_cut_rows_define`, `A_rebuild_reports_a_full_index_once_though_an_object_stream_lies_before_the_header`; `ReaderLimitsTests.A_catalog_packed_in_an_object_stream_past_a_full_index_reads_as_null_and_rebuilds_nothing`, `A_row_refused_before_a_full_index_is_the_file_s_and_an_object_stream_past_it_stays_unknown`; `DiagnosticsTests.A_guard_reached_is_counted_though_the_capacity_drops_its_report_and_follows_it_when_moved_or_rolled_back` | Yes | | The rebuild's scan | 1 MB pieces overlapping by 64 bytes, each piece after the first searched from where the one before could no longer hold a whole `obj`, so that a header is read once. A header is read backwards, each run of digits judged by its value however many zeros lead it, as the parser judges it: a number in 1..`int.MaxValue` and a generation in 0..65,535, which the entry records; the search near an entry's offset, a load's, settling's and the validator's probe's, reads a header so too. The definitions it finds again of a number are counted, with the first ten numbers, in a fixed-size record, and reported once as `object.redefined` | `DocumentReaderTests.Rebuilds_an_index_whose_objects_lie_past_the_first_megabyte`, `A_rebuild_ignores_object_headers_numbered_zero_or_beyond_what_an_object_number_holds`, `A_rebuild_finds_a_header_whose_number_and_generation_zeros_lead`, `A_rebuilt_entry_records_the_generation_its_header_gives`, `Counts_two_definitions_the_scan_finds_in_the_overlap_of_two_windows_as_one_redefinition`, `Reads_a_header_that_straddles_the_start_of_a_scan_window_under_its_own_number`; `HostileInputTests.Survives_a_file_made_only_of_object_headers`; `CrossReferenceRuleTests.An_entry_at_a_header_the_parser_refuses_is_broken` | Partly | | A catalog a guard cut — its `/Type` or its `/Pages` past the cut —, which leads to no catalog as read | The object `/Root` names, through a chain of references too, is taken for the catalog as read — one an object stream's cut leaves unread, past the cut data or in a stream the guard made unreadable, as one it cut ([#259]), and one written in a trailer `MaxTrailerLength` cut —, at opening and after any rebuild: no search among the indexed objects, which could take a catalog an update superseded, and no rebuild, which would read it cut again; `FileStructure.RootCutByGuard` records it, and `file.root-invalid` does not judge it ([#175]). A rebuild after a search reads the file's own `/Root` again, not the search's guess, and a guard it reaches there is held, as any in a rebuild. A rebuild keeps in its index the members the chain's index places in an object stream a guard hides — the scan finds no header of theirs —, which stay unread for the guard's reasons and throw again when opened to throw; a stream refused for a fault of its dictionary, read whole, hides none, whatever a limit cut of its data | `ReaderLimitsTests.A_catalog_a_guard_cut_is_taken_as_read_and_no_other_is_looked_for_even_by_a_later_rebuild`, `A_catalog_an_object_stream_s_cut_leaves_unread_stays_taken_as_read_through_a_rebuild`, `A_catalog_written_in_a_trailer_a_guard_cut_is_taken_as_read`, `A_catalog_written_in_a_trailer_a_guard_cut_stays_taken_as_read_through_a_rebuild`, `A_member_an_object_stream_s_cut_leaves_unread_throws_again_after_a_rebuild`, `An_object_stream_refused_for_its_own_fault_hides_no_member_where_a_limit_also_cut_its_data`, `A_rebuild_after_a_search_reads_the_file_s_own_root_again`, `A_rebuild_reaching_a_guard_on_a_catalog_a_search_took_still_looks_for_another_when_opened_to_throw`, `A_root_that_leads_through_a_chain_to_a_catalog_a_guard_cut_is_taken_as_read`, `A_catalog_an_object_stream_s_cut_leaves_unread_is_taken_as_read`; `FileRuleTests.A_catalog_a_limit_cut_inside_its_type_is_not_judged`, `A_catalog_a_limit_cut_before_its_type_is_taken_as_read_rather_than_an_older_one`, `A_catalog_that_is_none_is_still_judged_beside_one_a_limit_cut_elsewhere` | No | diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 56e55f15..9815e60c 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -4892,9 +4892,10 @@ private void ReserveIndex(long rows) var most = _guard.Bound(PdfLimit.IndexEntries); var room = (int)Math.Min(rows, int.MaxValue); - // The /Size as written, which ISO 32000-1 makes direct (Table 15, 7.5.8.2): a reference is not read through to make - // room by, as the newest cross-reference stream's beside an /Index, which numbers no row, would be while the chain is. - if (_xref.Trailer.GetRaw(PdfName.Size) is PdfInteger { Value: var size }) + // The /Size as written, which ISO 32000-1 makes direct (Table 15, 7.5.8.2), an integral real read as the integer it + // is: a reference is not read through to make room by, as the newest cross-reference stream's beside an /Index, which + // numbers no row, would be while the chain is. + if (_xref.Trailer.GetRaw(PdfName.Size) is { } written and not PdfReference && written.AsInteger() is { } size) { // A size the index holds as many entries as already asks for no room, nor does one below zero, which is not // narrowed to an int that would. One the sections read so far number past is stale, and they stand for it. diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index f10fc5c3..f48c1925 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -2725,15 +2725,21 @@ public void A_newest_size_far_past_the_room_made_so_far_is_not_made_room_for_at_ document.Reader.Index.Capacity.Should().BeLessThan(4000, "the map doubles from the newest section's 1,103 slots"); } - [Fact] - public void An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers() + [Theory] + [InlineData("3000")] + [InlineData("3000.0")] + public void An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers(string size) { // The update rewrites a thousand of the original's 3,000 objects: the original's rows for them supersede nothing, - // and room made for all its rows on top of the update's doubled the map. The newest /Size numbers every object. - var file = TestPdfBuilder.AppendIncrementalUpdate( + // and room made for all its rows on top of the update's doubled the map. The newest /Size numbers every object, + // written direct, as an integer or as a real with no fractional part, which the reader reads as the integer it is. + var built = TestPdfBuilder.AppendIncrementalUpdate( ChainOfSections(sections: 1, rows: 3000), rootNumber: 1, [.. Enumerable.Range(3, 1000).Select(number => (number, "(updated)"))]); + var text = Encoding.Latin1.GetString(built); + var newest = text.LastIndexOf("/Size 3000", StringComparison.Ordinal); + var file = Encoding.Latin1.GetBytes(string.Concat(text.AsSpan(0, newest), "/Size " + size, text.AsSpan(newest + "/Size 3000".Length))); using var document = PdfDocument.Open(file); From 283626b6fa1c5ac9902565adb14d04cf22bbab68 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 15:11:27 +0000 Subject: [PATCH 110/146] fix: name the object read as a cross-reference stream in its trailer guard's report The trailer guard reached where the chain reads a cross-reference stream said a cross-reference stream's dictionary ran past the bound, though what is there may be a font whose /W the reading took for a section's, then found no section. The report now names what it read: the dictionary of the object read as a cross-reference stream. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 2 +- tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 1 + tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 9815e60c..7a3590ec 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -5477,7 +5477,7 @@ private static void MarkCutByGuard(PdfObject parsed, long windowEnd) private static string LimitSubject(int number, int bound) => number switch { DirectObject => $"A trailer runs past {PdfLimitGuard.FormatLength(bound)}; only what lies within it was read.", - AnyObject => $"A cross-reference stream's dictionary runs past {PdfLimitGuard.FormatLength(bound)}; only what lies within it was read.", + AnyObject => $"The dictionary of the object read as a cross-reference stream runs past {PdfLimitGuard.FormatLength(bound)}; only what lies within it was read.", _ => string.Create( CultureInfo.InvariantCulture, $"Object {number} runs past {PdfLimitGuard.FormatLength(bound)}, its stream data aside; only what lies within it was read."), diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 265cedfd..887b3391 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -2307,6 +2307,7 @@ public void A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_n using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxTrailerLength = 128 } }); document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitTrailer, PdfDiagnosticCodes.XRefOffsetAdjusted); + document.Diagnostics[0].Message.Should().StartWith("The dictionary of the object read as a cross-reference stream runs past 128 bytes"); document.Reader.Structure.Sections[^1].Offset.Should().Be(stream); document.GetObject(new PdfObjectId(1)).AsDictionary().IsOfType(PdfName.Catalog).Should().BeTrue(); document.WasRepaired.Should().BeFalse(); diff --git a/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs b/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs index 82d291fa..f5d35a55 100644 --- a/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs +++ b/tests/AdCodicem.Pdf.Tests/ReaderLimitsTests.cs @@ -3760,7 +3760,7 @@ private static Case LongXRefStreamDictionary() PdfReaderLimits.Default with { MaxTrailerLength = 1024 * 1024 }, IndexState, "as written", - "A cross-reference stream's dictionary runs past 64 KB; only what lies within it was read.", + "The dictionary of the object read as a cross-reference stream runs past 64 KB; only what lies within it was read.", OffsetOf(file, "<< /Type /XRef") - "4 0 obj\n".Length, ThrowsOnOpening: true, ReportsNothingElse: false); From 49e599da7014012022fc673a03f66822589adc5d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 15:12:00 +0000 Subject: [PATCH 111/146] docs: give the read-through and the links the reader follows where the threat model omits them The threat model said a value read through a reference to an object a guard cut is the guard's, whatever the cut left, leaving out the whole reference the cut leaves after the value, which is read through as a load reads it. Two places still said any /Prev or /XRefStm that leads to such an object ends the chain, where only one the reader follows does. The trailer rule says why Table 8 does not govern a dictionary given as /Filter. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 5 +++-- docs/website/docs/concepts/lazy-reading.md | 2 +- src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index 9f72867e..087521d7 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -114,7 +114,8 @@ that meets the cut too ([#170]). What a guard leaves out of the index reads as null until it is raised, and the index is not rebuilt for it: a guard's cut is no damage. A value read through a reference to an object a guard cut or left out of the index, through a chain of references too, is the guard's as well, whatever the cut left of it — null, or a token it parted —, and the reader -judges none of it as the file's. That holds for a stream's `/Length` ([#224]) and the values its filters read +judges none of it as the file's — but for a whole reference the cut left after the value, which is read through as a +load reads it, what it leads to judged as read whole. That holds for a stream's `/Length` ([#224]) and the values its filters read ([#263]), for a cross-reference stream's `/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, that `MaxObjectLength` cut as the chain was read, and its `/Length` when no `endstream` ends its data, and for a `/Prev` or `/XRefStm` the reader follows that leads to such an object (*Indexing*); it adds no bound. When the document was opened with @@ -282,7 +283,7 @@ Known gaps: | A classic table of any length | Its window grows ×4 from 64 KB up to `MaxXRefSectionLength`, and the rows read are kept. A table the guard cuts, or whose rows run past a full index, keeps its trailer, found past the rows left unread by a search a megabyte at a time (`ReadTrailerPastRows`): comments are skipped, and anything else no row holds is the section's fault, reported, the search going on to the `trailer` keyword, so that its `/Encrypt` still refuses the document. The search starts at the start of the line the cut fell in, outside any comment and past the table's own keyword; it reads each megabyte once, every byte of it looked at before the next is read, however many comments or stray bytes it holds, and is not made for a place a section may have been moved to, a guess. The chain ends there, since an older section's rows would stand for the objects the unread rows define. What those rows place reads as null until the guard is raised | `ReaderLimitsTests.A_table_whose_keywords_the_bound_cuts_reports_the_bound`, `A_table_its_length_bound_cuts_keeps_its_trailer_and_the_encryption_it_names`, `A_table_its_length_bound_cuts_ends_the_chain_though_its_trailer_names_an_older_section`, `What_ends_the_rows_a_length_bound_cut_is_judged_as_it_is_where_none_did`, `A_table_a_length_bound_cuts_that_runs_to_the_end_of_the_file_has_no_trailer`, `A_table_its_length_bound_cuts_keeps_the_encryption_its_trailer_names_past_what_no_row_holds`, `A_table_cut_before_its_keyword_ends_keeps_its_trailer`, `Rows_past_a_full_index_are_not_read_and_the_trailer_past_them_is`, `A_cut_in_a_comment_among_the_rows_searches_for_the_trailer_from_the_comment_s_start`, `The_trailer_past_rows_left_unread_is_found_wherever_the_search_s_window_ends`, `A_trailer_past_rows_left_unread_that_its_own_bound_cuts_keeps_what_precedes_the_cut`; `DocumentReaderTests.The_search_for_the_trailer_past_rows_left_unread_reads_the_file_once`; `CrossReferenceChainTests.Reads_an_object_a_length_bound_cut_out_of_its_table_as_null_until_the_guard_is_lifted`, `A_section_found_near_where_prev_names_it_is_not_searched_past_the_rows_a_guard_left_unread`; `WindowEdgeTests.A_classic_table_reads_whole_wherever_its_window_edge_falls_near_its_trailer` | No: seeds stay under 120 KB | | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). A hybrid file's stream's own `/Prev`, which the chain does not follow, going on through the table's, is not read at all, nor are the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place: neither names a section missing nor ends the chain for a guard, and the rule on trailer values judges them as written, a reference they hold not followed ([#263]). An object the guard cut after a whole reference it holds hides nothing: the chain reads through it, as a load does, and loads it once the chain is read. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s`, `A_table_an_xrefstm_names_in_a_stream_s_place_has_its_own_links_left_unread`, `A_prev_written_as_a_reference_to_a_real_is_judged_through_it_only_where_the_reader_follows_it` | Yes | -| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); so is one that declared itself a section before a guard cut it — a table's `xref` keyword, `/Type /XRef`. A place that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is a guess dropped, the next place tried; where the chain names a section, a guard that cut what declares another type cut no section, and relocation is tried as read whole. A section that numbers its rows wrongly in what it writes direct — neither `/Size` nor `/Index` — is the file's fault, whatever a value a guard cut hides | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_cross_reference_stream_whose_deferred_length_a_guard_cut_is_found_where_it_was_moved`, `A_cross_reference_stream_a_guard_cut_the_dictionary_of_after_its_type_is_the_section_where_it_was_moved`, `A_classic_table_a_guard_cut_the_rows_of_is_the_section_where_it_was_moved`, `A_stream_with_no_numbering_whose_width_a_guard_cut_is_no_section_the_guard_cut`, `A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild` | No | +| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); so is one that declared itself a section before a guard cut it — a table's `xref` keyword, `/Type /XRef`. A place that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is a guess dropped, the next place tried; where the chain names a section, a guard that cut what declares another type cut no section, and relocation is tried as read whole. A section that numbers its rows wrongly in what it writes direct — neither `/Size` nor `/Index` — is the file's fault, whatever a value a guard cut hides | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_cross_reference_stream_whose_deferred_length_a_guard_cut_is_found_where_it_was_moved`, `A_cross_reference_stream_a_guard_cut_the_dictionary_of_after_its_type_is_the_section_where_it_was_moved`, `A_classic_table_a_guard_cut_the_rows_of_is_the_section_where_it_was_moved`, `A_stream_with_no_numbering_whose_width_a_guard_cut_is_no_section_the_guard_cut`, `A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild` | No | | A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, stays within the bound: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | diff --git a/docs/website/docs/concepts/lazy-reading.md b/docs/website/docs/concepts/lazy-reading.md index 8f8f48a0..af3a4aa4 100644 --- a/docs/website/docs/concepts/lazy-reading.md +++ b/docs/website/docs/concepts/lazy-reading.md @@ -62,7 +62,7 @@ sections past `MaxXRefSectionCount`, the rows past `MaxXRefSectionLength` or pas or `/DecodeParms`, or `/Size` with no `/Index`, leads to an object `MaxObjectLength` cuts and the sections older than it — so too one whose `/Length` leads to such an object, when no `endstream` ends its data before the next object or the end of the file, or the rows past an `endstream` its rows spell, which only that length could tell from the end of its data —, the sections -a `/Prev` or an `/XRefStm` that leads to such an object names — the older ones, or the cross-reference stream of a hybrid +a `/Prev` or an `/XRefStm` the reader follows that leads to such an object names — the older ones, or the cross-reference stream of a hybrid file's revision, with the sections older than it, the chain ending with it —, the numbers past `MaxIndexEntries`. An object the guard cut after a whole reference it holds hides none of them: the chain reads through it. That is no damage, and the map is not rebuilt for it. The map is rebuilt by scanning the whole file, once at most, and never while diff --git a/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs b/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs index 9948a73a..0505d1c8 100644 --- a/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs +++ b/src/AdCodicem.Pdf/Validation/Rules/TrailerValueWrongRule.cs @@ -204,8 +204,9 @@ public void Integers(PdfDictionary trailer, PdfName key) /// /// Reports a cross-reference stream's /Filter or /DecodeParms written as a reference, and, in a file /// that declares PDF 2.0, each element and entry written as one; and the integers of Table 8 a /DecodeParms - /// gives written as reals — not a dictionary a /Filter gives, which ends decoding unread. A reference among - /// them is followed where decoding follows it, as says. + /// gives written as reals — not a dictionary a /Filter gives, which is no decode parameters, Table 8 not + /// governing it, and ends decoding unread. A reference among them is followed where decoding follows it, as + /// says. /// public void Decoding(PdfDictionary trailer, bool pdf20, DecodingReads reads) { From 087ca76c4aa5c8a1ec80c46ecb2f6319c8462328 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 18:06:43 +0000 Subject: [PATCH 112/146] fix: decode as read whole data a proven endstream ends past white space Under a /Length a guard hid, data whose endstream is proven its own past white space was decoded as data a guard cut: an ASCII85 or ASCIIHex group it held in part was dropped and a predictor's last row undone. White space parts no such group and falls in a Flate checksum, so nothing was there to drop, and the member the drop shortened in an object stream was charged to the file: (wor) for (world), or a member past the data, the stream broken. The data now decodes as it would read whole. Its end only silences what a filter says of running out there: a missing end marker, a Flate tail or checksum, an LZW end-of-data code. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 2 +- .../IO/Filters/PdfFilterPipeline.cs | 30 +++++--- src/AdCodicem.Pdf/Objects/PdfStreamData.cs | 5 +- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 72 +++++++++++++++++++ 4 files changed, 96 insertions(+), 13 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index 087521d7..7141a7d9 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -355,7 +355,7 @@ Known gaps: | A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | -| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`), so that a Flate checksum short of a carriage return, or a group a filter holds in part, says nothing of the file. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | +| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`), so that a filter that runs out there — a Flate checksum short of a carriage return, an end marker missing — says nothing of the file, while the data decodes as read whole: white space parts no ASCII85 or ASCIIHex group, and no group is dropped nor a predictor's row undone for it. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | | Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, about the file once more; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | diff --git a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs index d8e4a782..e0931b1e 100644 --- a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs +++ b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs @@ -152,12 +152,15 @@ private static ReadOnlyMemory DecodeChain( // Whether the data a filter is given ends where the file's may not: data the reader cut at one of its limits — // MaxStreamLength, or the window a guard stopped at, cut the stream's data, or a filter before it stopped at - // MaxDecodedStreamLength —, or data whose end a guard hides: the caller says so, or the data does, its endstream - // proven its own past white space only a hidden length says is the data's or not. Every filter after it is given - // such data, and a cut is reported once, by the guard that made it; data whose end is hidden is no cut. A filter - // drops a last group such data holds in part, and counts it against the bound all the same (Ascii85Filter, - // AsciiHexFilter). - var cut = stream.Data.CutByGuard || endHidden || stream.Data.EndHidden; + // MaxDecodedStreamLength —, or data whose end the caller says a guard hides. Every filter after it is given such + // data, and a cut is reported once, by the guard that made it; data whose end is hidden is no cut. A filter drops a + // last group such data holds in part, and counts it against the bound all the same (Ascii85Filter, AsciiHexFilter). + var cut = stream.Data.CutByGuard || endHidden; + + // Data whose endstream is proven its own past white space only a hidden length says is the data's or not ends where + // the file's does but for that white space, which parts no ASCII85 or ASCIIHex group and falls in a Flate stream's + // checksum: it decodes as it would read whole, and a filter that runs out there says nothing of the file. + var silenced = stream.Data.EndHidden; // A guard a decoding of the same data reached before was reported then: its verdict stands, whatever this one meets. var limitReported = reachedBefore; @@ -188,6 +191,7 @@ private static ReadOnlyMemory DecodeChain( stream.Data.Position, guard, ref cut, + silenced, ref limitReported, out var failed); @@ -228,6 +232,10 @@ public static bool IsImageFilter(PdfName name) => /// /// Whether is data the reader cut; set when this filter stops at the bound too. /// + /// + /// Whether where the data ends says nothing of the file, though it is no cut: a filter that runs out there reports + /// nothing, and decodes as it would read whole. + /// /// Whether the stream reported a filter that stopped at the bound already. /// Whether the filter could decode nothing, which ends the chain. private static ReadOnlyMemory ApplyOne( @@ -238,12 +246,14 @@ private static ReadOnlyMemory ApplyOne( long position, PdfLimitGuard guard, ref bool cut, + bool silenced, ref bool limitReported, out bool failed) { var name = step.Name; var maxLength = bounds.MaxLength; var given = cut; + var quiet = given || silenced; failed = false; if (name == PdfName.FlateDecode) @@ -265,7 +275,7 @@ private static ReadOnlyMemory ApplyOne( // may: that it ran out says nothing of the file. A fault, or a checksum that disagrees, lies in bytes the reader // did read, and the whole data would meet it at the same point: it is the file's. ReportEnding( - given && ending is FlateEnding.TailLost or FlateEnding.ChecksumMissing ? FlateEnding.Whole : ending, + quiet && ending is FlateEnding.TailLost or FlateEnding.ChecksumMissing ? FlateEnding.Whole : ending, decoded.Length, faultAt, data.Length, @@ -289,7 +299,7 @@ private static ReadOnlyMemory ApplyOne( $"An LZW stream uses code {undefinedCode}, which it has not defined; what decoded before it was kept."), position); } - else if (ending == DataEnding.Unmarked && !given) + else if (ending == DataEnding.Unmarked && !quiet) { // Data the reader cut ends early because the reader stopped reading it, and data whose end a guard hides // ends where the guard may: that no code ends it says nothing of the file. @@ -298,7 +308,7 @@ private static ReadOnlyMemory ApplyOne( "An LZW stream ends without its end-of-data code; it decoded to the end of its data.", position); } - else if (ending == DataEnding.UnmarkedInsideCode && !given) + else if (ending == DataEnding.UnmarkedInsideCode && !quiet) { diagnostics?.Warn( PdfDiagnosticCodes.FilterFailed, @@ -330,7 +340,7 @@ private static ReadOnlyMemory ApplyOne( // The end of data the reader cut, or whose end a guard hides, or of data the filter stopped decoding at the bound, is // the reader's: only what the filter read is the file's. - ReportFaults(faults, name, data.Length, textLimited, ended: !given && !textLimited, diagnostics, position); + ReportFaults(faults, name, data.Length, textLimited, ended: !quiet && !textLimited, diagnostics, position); Reached(textLimited, name, bounds.Shared, diagnostics, position, guard, ref cut, ref limitReported); return text; } diff --git a/src/AdCodicem.Pdf/Objects/PdfStreamData.cs b/src/AdCodicem.Pdf/Objects/PdfStreamData.cs index 9dc2edb7..9778f37a 100644 --- a/src/AdCodicem.Pdf/Objects/PdfStreamData.cs +++ b/src/AdCodicem.Pdf/Objects/PdfStreamData.cs @@ -34,8 +34,9 @@ public abstract class PdfStreamData /// /// Gets a value indicating whether where the data ends is the guard's to hide, though the data is the file's: an /// endstream proven the stream's own ended it under a /Length a guard hid, with white space before the - /// keyword that only the length could say is the data's or not (#224). Decoding then takes its end as one a guard hides, - /// and says nothing of the file there. + /// keyword that only the length could say is the data's or not (#224). Decoding then says nothing of the file where the + /// data ends, which it decodes as it would read whole: white space parts no ASCII85 or ASCIIHex group, and falls in a + /// Flate stream's checksum. /// internal virtual bool EndHidden => false; diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 887b3391..a11966f6 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -899,6 +899,78 @@ public void Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_ends guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries); } + [Theory] + [InlineData("cut", "(world)", "world")] + [InlineData("left out", "(world)", "world")] + [InlineData("cut", "(w)", "w")] + [InlineData("left out", "(w)", "w")] + public void An_ascii85_object_stream_that_lacks_its_end_marker_before_a_proven_endstream_decodes_as_read_whole( + string guard, string last, string text) + { + // Object stream 10's ASCII85 data lacks its ~> and ends inside a group, then an end-of-line and its endstream, which + // the next object proves its own, under a /Length a guard hid. White space parts no group: the data decodes as read + // whole. Its last group was dropped, as one a cut parts, and the member it shortened charged to the file — (wor), or + // (w) past the 16 bytes left, the object stream broken (#224). + var encoded = Ascii85Text(Encoding.Latin1.GetBytes("6 0 7 8 (hello) " + last)); + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = (1, file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), 0), + [2] = (1, file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), 0), + [3] = (1, file.Object(3, Page), 0), + [20] = (1, file.Object(20, (guard == "cut" ? new string(' ', 2000) : string.Empty) + encoded.Length.ToString(CultureInfo.InvariantCulture)), 0), + [10] = (1, file.Object(10, $"<< /Type /ObjStm /N 2 /First 8 /Length 20 0 R /Filter /ASCII85Decode >>\nstream\n{encoded}\nendstream"), 0), + [11] = (1, file.Object(11, "(eleven)"), 0), + [6] = (2, 10, 0), + [7] = (2, 10, 1), + }; + file.CrossReferenceStream(21, 22, rows, "/Root 1 0 R"); + var limits = guard == "cut" + ? PdfReaderLimits.Default with { MaxObjectLength = 1024 } + : PdfReaderLimits.Default with { MaxIndexEntries = 20 }; + + using (var whole = PdfDocument.Open(file.Bytes)) + { + whole.GetObject(new PdfObjectId(7)).Should().BeOfType().Which.ToText().Should().Be(text); + } + + using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = limits }); + + document.GetObject(new PdfObjectId(7)).Should().BeOfType().Which.ToText().Should().Be(text); + document.GetObject(new PdfObjectId(10)).AsStream().Required().Data.EndHidden.Should().BeTrue(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal( + guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries); + new PdfValidator().Validate(document).Contains(PdfValidationRuleIds.XRefObjectStreamBroken).Should().BeFalse(); + } + + /// Writes in ASCII85, whole groups but the last, with no z and no ~>. + private static string Ascii85Text(ReadOnlySpan bytes) + { + var text = new StringBuilder(); + Span digits = stackalloc char[5]; + + for (var i = 0; i < bytes.Length; i += 4) + { + var count = Math.Min(4, bytes.Length - i); + uint value = 0; + + for (var j = 0; j < 4; j++) + { + value = (value << 8) | (j < count ? bytes[i + j] : 0u); + } + + for (var j = 4; j >= 0; j--) + { + digits[j] = (char)('!' + (value % 85)); + value /= 85; + } + + text.Append(digits[..(count + 1)]); + } + + return text.ToString(); + } + private static byte[] ZLibDecompress(byte[] zlib) { using var input = new MemoryStream(zlib); From 52f1145bc204165d4e94473eb3b61f44e0d3a10b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 18:06:52 +0000 Subject: [PATCH 113/146] fix: count a group a cut parts against what the filters before the last share A group ASCII85 or ASCIIHex drops where data a guard cut or whose end is hidden parts it counted against the bound, but not against the allowance the filters before the last share: in a chain of three filters, the one after it was given the dropped bytes' room as well. Deferred data decoded again knowing its end hidden read a row past where its direct twin stopped: the page indexed where the twin leaves it to the guard, or a row and no rebuild where the twin rebuilds. The filters say how many bytes they dropped, and the allowance counts them. Refs #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 2 +- src/AdCodicem.Pdf/IO/Filters/Ascii85Filter.cs | 3 +- .../IO/Filters/AsciiHexFilter.cs | 3 +- src/AdCodicem.Pdf/IO/Filters/DataFaults.cs | 6 ++ .../IO/Filters/PdfFilterPipeline.cs | 20 +++-- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 73 +++++++++++++++++++ .../DocumentReaderTests.cs | 31 ++++++++ 7 files changed, 129 insertions(+), 9 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index 7141a7d9..48d03848 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -284,7 +284,7 @@ Known gaps: | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). A hybrid file's stream's own `/Prev`, which the chain does not follow, going on through the table's, is not read at all, nor are the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place: neither names a section missing nor ends the chain for a guard, and the rule on trailer values judges them as written, a reference they hold not followed ([#263]). An object the guard cut after a whole reference it holds hides nothing: the chain reads through it, as a load does, and loads it once the chain is read. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s`, `A_table_an_xrefstm_names_in_a_stream_s_place_has_its_own_links_left_unread`, `A_prev_written_as_a_reference_to_a_real_is_judged_through_it_only_where_the_reader_follows_it` | Yes | | What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); so is one that declared itself a section before a guard cut it — a table's `xref` keyword, `/Type /XRef`. A place that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is a guess dropped, the next place tried; where the chain names a section, a guard that cut what declares another type cut no section, and relocation is tried as read whole. A section that numbers its rows wrongly in what it writes direct — neither `/Size` nor `/Index` — is the file's fault, whatever a value a guard cut hides | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_cross_reference_stream_whose_deferred_length_a_guard_cut_is_found_where_it_was_moved`, `A_cross_reference_stream_a_guard_cut_the_dictionary_of_after_its_type_is_the_section_where_it_was_moved`, `A_classic_table_a_guard_cut_the_rows_of_is_the_section_where_it_was_moved`, `A_stream_with_no_numbering_whose_width_a_guard_cut_is_no_section_the_guard_cut`, `A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, stays within the bound: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, may stay within the bound — the group counted against what the filters before the last share as well —: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, written direct — an integer, or a real with no fractional part; one written as a reference is not read through for it, and bounds nothing —, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | diff --git a/src/AdCodicem.Pdf/IO/Filters/Ascii85Filter.cs b/src/AdCodicem.Pdf/IO/Filters/Ascii85Filter.cs index 32317651..5f795f9d 100644 --- a/src/AdCodicem.Pdf/IO/Filters/Ascii85Filter.cs +++ b/src/AdCodicem.Pdf/IO/Filters/Ascii85Filter.cs @@ -21,7 +21,7 @@ internal static class Ascii85Filter /// Whether the data ends where the file's may not — the reader cut it short at one of its limits, or a guard hides /// where it ends —: a group left partial at its end, before any ~, is one the cut parted, and is dropped rather /// than completed, though the bytes it encodes count against , which - /// then says. + /// then says, and counts them. /// public static byte[] Decode( ReadOnlySpan data, out bool limited, out DataFaults faults, int maxLength, bool cut = false) @@ -149,6 +149,7 @@ public static byte[] Decode( // Data that ends where the file's may not leaves a group it parts, whose bytes the missing digits decide: none is // written. The whole data holds at least the count-1 bytes it encodes, which the bound counts all the same. limited = count - 1 > maxLength - output.Count; + faults.Dropped = count - 1; } if (!ended) diff --git a/src/AdCodicem.Pdf/IO/Filters/AsciiHexFilter.cs b/src/AdCodicem.Pdf/IO/Filters/AsciiHexFilter.cs index 3f25545e..251fa9c3 100644 --- a/src/AdCodicem.Pdf/IO/Filters/AsciiHexFilter.cs +++ b/src/AdCodicem.Pdf/IO/Filters/AsciiHexFilter.cs @@ -21,7 +21,7 @@ internal static class AsciiHexFilter /// Whether the data ends where the file's may not — the reader cut it short at one of its limits, or a guard hides /// where it ends —: a digit left alone at its end, before any >, is the first of a pair the cut parted, and is /// dropped rather than completed with a zero, though the byte counts against , which - /// then says. + /// then says, and counts it. /// public static byte[] Decode( ReadOnlySpan data, out bool limited, out DataFaults faults, int maxLength, bool cut = false) @@ -93,6 +93,7 @@ public static byte[] Decode( // Data that ends where the file's may not leaves its last byte half read, which the next digit decides: it is // not written, and the whole data, which holds it, counts it against the bound all the same. limited = length >= maxLength; + faults.Dropped = 1; } return buffer.AsSpan(0, length).ToArray(); diff --git a/src/AdCodicem.Pdf/IO/Filters/DataFaults.cs b/src/AdCodicem.Pdf/IO/Filters/DataFaults.cs index 60fa20fa..30b143ba 100644 --- a/src/AdCodicem.Pdf/IO/Filters/DataFaults.cs +++ b/src/AdCodicem.Pdf/IO/Filters/DataFaults.cs @@ -46,6 +46,12 @@ internal struct DataFaults /// For data that ends inside a RunLength run: how many of its bytes it holds. public int UnitHeld; + /// + /// For data the reader cut that ends inside an ASCII85 group or an ASCIIHex pair: how many bytes the unit would decode + /// to, which were not written. + /// + public int Dropped; + /// Counts a byte passed over at , counted from 0. public void Pass(int index) { diff --git a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs index e0931b1e..60b3b088 100644 --- a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs +++ b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs @@ -70,10 +70,10 @@ public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diag /// where the length that would say is one a guard cut, or a guard on decoding kept the data from being taken again to /// it. The data decodes as data a guard cut does — a last group a filter holds in part, which the hidden end may have /// parted, is dropped, and a predictor's last row undone as far as it goes —, though it is not counted as cut by it: a - /// group dropped so counts against all the same, the - /// whole data holding at least as much. A filter after it is given less, and may stay within the bound where the whole - /// data would not: a decoding of data whose end is hidden that a decoding of it reached the bound before keeps that - /// verdict, with . + /// group dropped so counts against all the same, and + /// against what the filters before the last share, the whole data holding at least as much. A filter after it is given + /// less, and may stay within the bound where the whole data would not: a decoding of data whose end is hidden that a + /// decoding of it reached the bound before keeps that verdict, with . /// /// /// Whether a decoding of the same data stopped at before, @@ -154,7 +154,8 @@ private static ReadOnlyMemory DecodeChain( // MaxStreamLength, or the window a guard stopped at, cut the stream's data, or a filter before it stopped at // MaxDecodedStreamLength —, or data whose end the caller says a guard hides. Every filter after it is given such // data, and a cut is reported once, by the guard that made it; data whose end is hidden is no cut. A filter drops a - // last group such data holds in part, and counts it against the bound all the same (Ascii85Filter, AsciiHexFilter). + // last group such data holds in part, and counts it against the bound all the same (Ascii85Filter, AsciiHexFilter), + // and against what the filters before the last share. var cut = stream.Data.CutByGuard || endHidden; // Data whose endstream is proven its own past white space only a hidden length says is the data's or not ends where @@ -193,6 +194,7 @@ private static ReadOnlyMemory DecodeChain( ref cut, silenced, ref limitReported, + out var dropped, out var failed); // Data a filter could not decode at all comes back as the filters before it left it, as it does at a filter the @@ -203,7 +205,9 @@ private static ReadOnlyMemory DecodeChain( return data; } - shared -= data.Length; + // A group the filter dropped at the cut is data the whole would decode to, and leaves the filters after it no + // more room than the whole data would. + shared -= data.Length + dropped; } ReportEnd(chain, diagnostics, stream.Data.Position, guard); @@ -237,6 +241,7 @@ public static bool IsImageFilter(PdfName name) => /// nothing, and decodes as it would read whole. /// /// Whether the stream reported a filter that stopped at the bound already. + /// How many bytes a last group the cut parted would decode to, which the filter did not write. /// Whether the filter could decode nothing, which ends the chain. private static ReadOnlyMemory ApplyOne( in FilterStep step, @@ -248,12 +253,14 @@ private static ReadOnlyMemory ApplyOne( ref bool cut, bool silenced, ref bool limitReported, + out int dropped, out bool failed) { var name = step.Name; var maxLength = bounds.MaxLength; var given = cut; var quiet = given || silenced; + dropped = 0; failed = false; if (name == PdfName.FlateDecode) @@ -342,6 +349,7 @@ private static ReadOnlyMemory ApplyOne( // the reader's: only what the filter read is the file's. ReportFaults(faults, name, data.Length, textLimited, ended: !quiet && !textLimited, diagnostics, position); Reached(textLimited, name, bounds.Shared, diagnostics, position, guard, ref cut, ref limitReported); + dropped = faults.Dropped; return text; } diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 9970e66a..502ee873 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -1228,6 +1228,79 @@ public static byte[] UpdateHidingItsHighestNumber(int padding) return file.ToArray(); } + /// + /// A file of one revision whose cross-reference stream 5 numbers 40 rows, 39 the page's, and takes its /Length + /// from object 4 when , which only it places. Its rows pass through three filters, the first + /// lacking its end marker and ending inside a unit: with , + /// /Filter [/ASCIIHexDecode /RunLengthDecode /FlateDecode], the hexadecimal digits short of their last; + /// otherwise /Filter [/ASCII85Decode /FlateDecode /RunLengthDecode], the last ASCII85 group partial. RunLength + /// writes literal runs of 128 bytes. + /// + public static byte[] RowsThroughThreeFilters(bool deferred, bool hex) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [39 0 R] /Count 1 >>"); + file.Object(39, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + var four = file.Position; + var at = four + "4 0 obj\n000000\nendobj\n".Length; + var rows = new (int Type, long Second, int Third)[40]; + rows[0] = (0, 0, 65535); + rows[1] = (1, file.OffsetOf(1), 0); + rows[2] = (1, file.OffsetOf(2), 0); + rows[4] = (1, four, 0); + rows[5] = (1, at, 0); + rows[39] = (1, file.OffsetOf(39), 0); + var raw = Encode(rows, Rows.Raw); + byte[] data; + + if (hex) + { + var digits = Convert.ToHexString(RunLength(Deflate(raw))); + data = Encoding.ASCII.GetBytes(digits[..^1]); + } + else + { + data = Encoding.ASCII.GetBytes(Ascii85(Deflate(RunLength(raw)))); + } + + var length = data.Length.ToString("D6", CultureInfo.InvariantCulture); + file.Object(4, length); + file.Stream( + 5, + $"/Type /XRef /Size 40 /W [1 4 2] /Filter {(hex ? "[/ASCIIHexDecode /RunLengthDecode /FlateDecode]" : "[/ASCII85Decode /FlateDecode /RunLengthDecode]")} /Root 1 0 R /Length {(deferred ? "4 0 R" : length)}", + data); + file.StartXRef(at); + return file.ToArray(); + + static byte[] Deflate(byte[] bytes) + { + using var output = new MemoryStream(); + + using (var zlib = new ZLibStream(output, CompressionLevel.SmallestSize, leaveOpen: true)) + { + zlib.Write(bytes); + } + + return output.ToArray(); + } + + static byte[] RunLength(byte[] bytes) + { + var runs = new List(); + + for (var i = 0; i < bytes.Length; i += 128) + { + var count = Math.Min(128, bytes.Length - i); + runs.Add((byte)(count - 1)); + runs.AddRange(bytes.AsSpan(i, count)); + } + + runs.Add(128); + return [.. runs]; + } + } + private enum Rows { Raw, diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index f48c1925..53c2d03d 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -2005,6 +2005,37 @@ public void A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedSt deferred.Reader.Structure.Sections[^1].Incomplete.Should().BeFalse(); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Data_whose_end_a_guard_hides_leaves_the_filters_after_a_dropped_group_the_room_its_direct_twin_s_have(bool hex) + { + // Stream 5's rows pass through three filters, the first lacking its end marker and ending inside a unit. Decoded once + // more knowing its end hidden, the first filter drops the unit, whose bytes counted against MaxDecodedStreamLength + // but not against what the filters before the last share: the filter after it was given more room than the direct + // twin's, and read a row past where it stopped — the page indexed beside a guard where the direct twin leaves it out, + // or a row and no rebuild where the direct twin rebuilds (#220). + var direct = ChainFiles.RowsThroughThreeFilters(deferred: false, hex); + var deferred = ChainFiles.RowsThroughThreeFilters(deferred: true, hex); + + for (var bound = 1; bound <= 400; bound++) + { + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxDecodedStreamLength = bound } }; + using var directDocument = PdfDocument.Open(direct, options); + using var deferredDocument = PdfDocument.Open(deferred, options); + + Read(deferredDocument).Should().Be(Read(directDocument), $"MaxDecodedStreamLength is {bound}"); + } + + static string Read(PdfDocument document) => string.Join( + ' ', + document.WasRepaired, + document.Reader.Structure.Sections[^1].CutByLimit, + document.GetObject(new PdfObjectId(1)) is PdfDictionary, + document.GetObject(new PdfObjectId(39)) is PdfDictionary, + document.Reader.ChainIndex?.Count); + } + [Fact] public void A_deferred_length_s_data_that_holds_every_row_under_a_guard_on_decoding_is_checked_against_its_length() { From 462ded111a4c13d93487270108bfb07bbdfad774 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 18:19:55 +0000 Subject: [PATCH 114/146] fix: judge what a value a guard cut cannot change before charging a section to it A cross-reference stream a value's cut kept from being read was the guard's unless the numbering it wrote direct was wrong. A /W written wrong, a numbering wrong through a reference read whole, an /Index holding no pairs whatever its elements, or a value past the cut no section places makes the section unreadable whatever the guard hid, and read whole is the file's fault: the chain ended, the index was not rebuilt, the catalog was lost, and a real section was lost to a guessed place. Before a value's cut is taken for the guard's, the values past it are read on, and the widths and the numbering are judged as far as what was read says, in the whole read's order. A stream that declares another type is no section a guard cut a value of, as when the guard cuts its dictionary. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 6 +- docs/website/docs/reference/reader-limits.md | 8 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 318 ++++++++++++------ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 119 +++++++ 4 files changed, 351 insertions(+), 100 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index 48d03848..0d7d53dd 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -283,8 +283,8 @@ Known gaps: | A classic table of any length | Its window grows ×4 from 64 KB up to `MaxXRefSectionLength`, and the rows read are kept. A table the guard cuts, or whose rows run past a full index, keeps its trailer, found past the rows left unread by a search a megabyte at a time (`ReadTrailerPastRows`): comments are skipped, and anything else no row holds is the section's fault, reported, the search going on to the `trailer` keyword, so that its `/Encrypt` still refuses the document. The search starts at the start of the line the cut fell in, outside any comment and past the table's own keyword; it reads each megabyte once, every byte of it looked at before the next is read, however many comments or stray bytes it holds, and is not made for a place a section may have been moved to, a guess. The chain ends there, since an older section's rows would stand for the objects the unread rows define. What those rows place reads as null until the guard is raised | `ReaderLimitsTests.A_table_whose_keywords_the_bound_cuts_reports_the_bound`, `A_table_its_length_bound_cuts_keeps_its_trailer_and_the_encryption_it_names`, `A_table_its_length_bound_cuts_ends_the_chain_though_its_trailer_names_an_older_section`, `What_ends_the_rows_a_length_bound_cut_is_judged_as_it_is_where_none_did`, `A_table_a_length_bound_cuts_that_runs_to_the_end_of_the_file_has_no_trailer`, `A_table_its_length_bound_cuts_keeps_the_encryption_its_trailer_names_past_what_no_row_holds`, `A_table_cut_before_its_keyword_ends_keeps_its_trailer`, `Rows_past_a_full_index_are_not_read_and_the_trailer_past_them_is`, `A_cut_in_a_comment_among_the_rows_searches_for_the_trailer_from_the_comment_s_start`, `The_trailer_past_rows_left_unread_is_found_wherever_the_search_s_window_ends`, `A_trailer_past_rows_left_unread_that_its_own_bound_cuts_keeps_what_precedes_the_cut`; `DocumentReaderTests.The_search_for_the_trailer_past_rows_left_unread_reads_the_file_once`; `CrossReferenceChainTests.Reads_an_object_a_length_bound_cut_out_of_its_table_as_null_until_the_guard_is_lifted`, `A_section_found_near_where_prev_names_it_is_not_searched_past_the_rows_a_guard_left_unread`; `WindowEdgeTests.A_classic_table_reads_whole_wherever_its_window_edge_falls_near_its_trailer` | No: seeds stay under 120 KB | | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). A hybrid file's stream's own `/Prev`, which the chain does not follow, going on through the table's, is not read at all, nor are the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place: neither names a section missing nor ends the chain for a guard, and the rule on trailer values judges them as written, a reference they hold not followed ([#263]). An object the guard cut after a whole reference it holds hides nothing: the chain reads through it, as a load does, and loads it once the chain is read. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s`, `A_table_an_xrefstm_names_in_a_stream_s_place_has_its_own_links_left_unread`, `A_prev_written_as_a_reference_to_a_real_is_judged_through_it_only_where_the_reader_follows_it` | Yes | -| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); so is one that declared itself a section before a guard cut it — a table's `xref` keyword, `/Type /XRef`. A place that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is a guess dropped, the next place tried; where the chain names a section, a guard that cut what declares another type cut no section, and relocation is tried as read whole. A section that numbers its rows wrongly in what it writes direct — neither `/Size` nor `/Index` — is the file's fault, whatever a value a guard cut hides | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_cross_reference_stream_whose_deferred_length_a_guard_cut_is_found_where_it_was_moved`, `A_cross_reference_stream_a_guard_cut_the_dictionary_of_after_its_type_is_the_section_where_it_was_moved`, `A_classic_table_a_guard_cut_the_rows_of_is_the_section_where_it_was_moved`, `A_stream_with_no_numbering_whose_width_a_guard_cut_is_no_section_the_guard_cut`, `A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, may stay within the bound — the group counted against what the filters before the last share as well —: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file` | No | +| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); so is one that declared itself a section before a guard cut it — a table's `xref` keyword, `/Type /XRef`. A place that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is a guess dropped, the next place tried; where the chain names a section, a guard that cut what declares another type — its dictionary or a value of it — cut no section, and relocation is tried as read whole. What makes a section unreadable whatever a value a guard cut holds is the file's fault, as read whole, judged before the cut is taken for the guard's (`PdfFileReader.FaultBesideCut`): a value past the cut that no section read so far places, which the sections a whole read reads before make the file's to lack; a `/W` that gives no three widths, or none past 0; rows numbered wrongly — an `/Index` that holds no pairs, whatever its elements, a subsection out of range, a `/Size` with no `/Index` that gives no count —, as far as the values read, written direct or through references, say: a value a guard cut says nothing, nor does what depends on it, the `/Size` an `/Index` so cut may stand beside | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_cross_reference_stream_whose_deferred_length_a_guard_cut_is_found_where_it_was_moved`, `A_cross_reference_stream_a_guard_cut_the_dictionary_of_after_its_type_is_the_section_where_it_was_moved`, `A_classic_table_a_guard_cut_the_rows_of_is_the_section_where_it_was_moved`, `A_stream_with_no_numbering_whose_width_a_guard_cut_is_no_section_the_guard_cut`, `A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild`, `What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_guard_cut_holds_is_the_file_s`, `A_place_a_section_may_have_moved_to_that_a_value_past_a_guard_s_cut_makes_unreadable_is_dropped`, `A_stream_of_another_type_whose_value_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, may stay within the bound — the group counted against what the filters before the last share as well —: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `Data_whose_end_a_guard_hides_leaves_the_filters_after_a_dropped_group_the_room_its_direct_twin_s_have`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, written direct — an integer, or a real with no fractional part; one written as a reference is not read through for it, and bounds nothing —, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | @@ -355,7 +355,7 @@ Known gaps: | A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | -| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`), so that a filter that runs out there — a Flate checksum short of a carriage return, an end marker missing — says nothing of the file, while the data decodes as read whole: white space parts no ASCII85 or ASCIIHex group, and no group is dropped nor a predictor's row undone for it. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | +| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`), so that a filter that runs out there — a Flate checksum short of a carriage return, an end marker missing — says nothing of the file, while the data decodes as read whole: white space parts no ASCII85 or ASCIIHex group, and no group is dropped nor a predictor's row undone for it. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `An_ascii85_object_stream_that_lacks_its_end_marker_before_a_proven_endstream_decodes_as_read_whole`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | | Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, about the file once more; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index 2cd13f93..0c944da9 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -140,9 +140,11 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. offset it is named at, which such a value keeps from being read, or which declared itself one — a table's `xref` keyword, `/Type /XRef` — before a guard cut it; a place there that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is not taken for the section, and the next is - tried. A guard that cut, where the chain names a section, what declares another type cut no section; and a section - that numbers its rows wrongly in what it writes direct — neither `/Size` nor `/Index` — is the file's fault, whatever a - value a guard cut hides. The rows a cross-reference stream's data lacks, + tried. A guard that cut, where the chain names a section, what declares another type — its dictionary or a value of it + — cut no section; and what makes a section unreadable whatever a value a guard cut holds is the file's fault, as read + whole: a value past the cut that no section read so far places, a `/W` that gives no three widths or none past 0, or + rows numbered wrongly — an `/Index` that holds no pairs, or a subsection, or a `/Size` with no `/Index`, out of + range —, as far as the values read, written direct or through references, say. The rows a cross-reference stream's data lacks, where its `/Length` so leads and its data was taken to the first `endstream`, are the guard's too, unless what follows that `endstream` proves it the stream's own, as above: they are then the file's to lack, and the index is rebuilt. An object the chain read through a reference the cut left whole is loaded once the chain is read only for a value it diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 7a3590ec..3766989d 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3792,11 +3792,12 @@ private bool ReadSubsection(ref PdfLexer lexer, int first, int count, XRefSectio /// /// An object that declares itself a cross-reference stream — /Type /XRef, or a /W, beside no other /// /Type at a place the section may have been moved to — and cannot be read is a malformed section; any - /// other object there, or none, is no section at all, and a guard that cut what declares another type cut no - /// section. What it writes direct that numbers its rows wrongly is the file's fault, whatever a value a guard cut - /// hides. At a place the section may have been moved to (), nothing such an object's - /// reading met is reported or recorded — its search for an endstream aside, which is kept as every search is - /// —: it was read as a section, through a section's window, and is read as itself when something asks for it. + /// other object there, or none, is no section at all, and a guard that cut what declares another type, its dictionary + /// or a value of it, cut no section. What makes it unreadable whatever a value a guard cut holds is the file's fault, + /// as read whole (). At a place the section may have been moved to + /// (), nothing such an object's reading met is reported or recorded — its search for an + /// endstream aside, which is kept as every search is —: it was read as a section, through a section's window, + /// and is read as itself when something asks for it. /// /// /// ISO 32000-1 (7.5.8.2) makes the values of this dictionary direct, the elements of /W and /Index @@ -3901,18 +3902,13 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect if (cutObject is { } past) { - // What the section writes direct that makes it unreadable whatever the guard hid is the file's, as read whole. - if (WrittenNumberingFault(dictionary) is { } numberingWritten) - { - section.Fault = numberingWritten; - return XRefSectionState.Malformed; - } - - section.Fault = lengthFault; - section.CutByLimit = true; - section.CutByValue = true; - (_chainValuesCut ??= []).Add(past); - return XRefSectionState.Malformed; + // What makes the section unreadable whatever the length the guard hid holds is the file's, as read whole: a + // value no section read so far places, or what the values read give. + var file = UnreadValue(dictionary, out var unreadCut) is { } unplaced && unreadCut is null + ? unplaced + : FaultBesideCut(dictionary); + section.Fault = file ?? lengthFault; + return file is null ? CutByValue(section, value, past) : XRefSectionState.Malformed; } var where = search.NextObject is { } next @@ -3924,34 +3920,31 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect return XRefSectionState.Malformed; } - if (UnreadValue(dictionary, out var unreadCut) is { } unread) + if (UnreadValue(dictionary, out var valueCut) is { } unread) { - // A value a guard cut is the guard's, as rows a guard cut are: the section counts as cut, the chain ends here, - // and nothing is rebuilt for it (ADR 34). The guard is reached once the chain is read (#263). section.Fault = unread; - if (unreadCut is { } cut) + // A value no section read so far places is the file's fault. One a guard cut is the guard's, as rows a guard cut + // are (CutByValue), but for what makes the section unreadable whatever the value holds: the file's, as read whole. + if (valueCut is not { } cut) { - // What the section writes direct that makes it unreadable whatever the guard hid is the file's, as read whole. - if (WrittenNumberingFault(dictionary) is { } numberingWritten) - { - section.Fault = numberingWritten; - return XRefSectionState.Malformed; - } + return XRefSectionState.Malformed; + } - section.CutByLimit = true; - section.CutByValue = true; - (_chainValuesCut ??= []).Add(cut); + if (FaultBesideCut(dictionary) is { } file) + { + section.Fault = file; + return XRefSectionState.Malformed; } - return XRefSectionState.Malformed; + return CutByValue(section, value, cut); } var widths = dictionary.GetArray(PdfName.W); if (widths is null || widths.Count < 3) { - section.Fault = "its /W does not give the widths of three fields"; + section.Fault = WidthsMissing; return XRefSectionState.Malformed; } @@ -3964,25 +3957,19 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect { var width = widths.Resolved(i).AsInteger(); - if (width is null) + if (WidthFault(width) is { } widthFault) { - section.Fault = "its /W gives a field a width that is not an integer"; + section.Fault = widthFault; return XRefSectionState.Malformed; } - if (width < 0) - { - section.Fault = "its /W gives a field a negative width"; - return XRefSectionState.Malformed; - } - - declaredWidths[i] = width.Value; + declaredWidths[i] = width.GetValueOrDefault(); anyWidth |= width > 0; } if (!anyWidth) { - section.Fault = "its /W gives rows of no bytes"; + section.Fault = NoRowBytes; return XRefSectionState.Malformed; } @@ -4389,41 +4376,118 @@ private static bool DeclaresCrossReferenceStream(PdfObject value) => (dictionary.IsOfType(PdfName.XRef) || dictionary.ContainsKey(PdfName.W)); /// - /// Says what is wrong with how a cross-reference stream numbers its rows, as far as it writes it direct: an - /// /Index or /Size written as a reference, or an /Index element, may lead to what a guard cut, and - /// says nothing then. Null when nothing written direct is wrong. + /// Takes a section a value's cut keeps from being read for the guard's: the section counts as cut, the chain ends there + /// and nothing is rebuilt for it (ADR 34), and the object the guard cut is loaded once the chain is read, which reaches + /// the guard (#263). What declares another type is no section a guard cut, a font's /W giving widths: it is the + /// file's to be, as read whole. /// - private static string? WrittenNumberingFault(PdfDictionary dictionary) + /// The section, whose fault names the value. + /// What the offset holds. + /// The object the guard cut that the value leads to. + private XRefSectionState CutByValue(XRefSectionRecord section, PdfObject value, PdfObjectId cut) { - var written = dictionary.GetRaw(PdfName.Index); - - if (written is PdfReference) + if (DeclaresOnlyCrossReferenceStream(value)) { - return null; + section.CutByLimit = true; + section.CutByValue = true; + (_chainValuesCut ??= []).Add(cut); } - if (written is PdfArray ranges) + return XRefSectionState.Malformed; + } + + /// + /// Says what makes a cross-reference stream unreadable whatever the values a guard cut hold, as a whole read says it: + /// the widths its /W gives, then how it numbers its rows, as far as the values read say. A value a guard cut, or + /// an element of one, says nothing, nor does what depends on it — the /Size an /Index so cut may stand + /// beside —; an /Index that holds no pairs is wrong whatever its elements hold. Null when nothing read is wrong: + /// the section is then the guard's. + /// + /// + /// followed every value read here, and found none that no section read so far places: a value + /// that cannot be read is one a guard cut. + /// + private string? FaultBesideCut(PdfDictionary dictionary) + { + if (Unread(dictionary, PdfName.W, out var w, out _) is null) { - foreach (var element in ranges) + if (w is not PdfArray widths || widths.Count < 3) { - if (element is PdfReference) + return WidthsMissing; + } + + var known = true; + var anyWidth = false; + + for (var i = 0; i < 3; i++) + { + if (Follow(PdfName.W, widths[i], entry: false, out var element, out _) is not null) { - return null; + known = false; + continue; } + + var width = element?.AsInteger(); + + if (WidthFault(width) is { } fault) + { + return fault; + } + + anyWidth |= width > 0; + } + + if (known && !anyWidth) + { + return NoRowBytes; } + } + + if (Unread(dictionary, PdfName.Index, out var index, out _) is not null) + { + return null; + } + + if (index is null or PdfNull) + { + return Unread(dictionary, PdfName.Size, out var size, out _) is null + ? NumberingFault(null, null, size?.AsInteger(), out _) + : null; + } - return NumberingFault(written, written, null, out _); + if (index is not PdfArray ranges || PairsFault(ranges) is not null) + { + return NumberingFault(dictionary.GetRaw(PdfName.Index), index, null, out _); } - if (written is not (null or PdfNull)) + for (var i = 0; i < ranges.Count; i += 2) { - return NumberingFault(written, written, null, out _); + if (Follow(PdfName.Index, ranges[i], entry: false, out var start, out _) is null && + Follow(PdfName.Index, ranges[i + 1], entry: false, out var count, out _) is null && + SubsectionFault(ranges, i, start?.AsInteger(), count?.AsInteger()) is { } fault) + { + return fault; + } } - var size = dictionary.GetRaw(PdfName.Size); - return size is PdfReference ? null : NumberingFault(null, null, size?.AsInteger(), out _); + return null; } + /// The fault of a /W that is no array of three widths. + private const string WidthsMissing = "its /W does not give the widths of three fields"; + + /// The fault of a /W whose widths are all 0. + private const string NoRowBytes = "its /W gives rows of no bytes"; + + /// Says what is wrong with a width a /W gives, or null when nothing is. + /// The width, when it is an integer. + private static string? WidthFault(long? width) => width switch + { + null => "its /W gives a field a width that is not an integer", + < 0 => "its /W gives a field a negative width", + _ => null, + }; + /// /// Determines whether what a place a section may have been moved to holds declares itself a cross-reference stream, and /// no other type: /Type /XRef, or a /W beside no /Type that names another. A guess is no section @@ -4449,45 +4513,91 @@ private static bool DeclaresOnlyCrossReferenceStream(PdfObject value) => /// /// The cross-reference stream's dictionary. /// The object a guard cut that the value which cannot be read leads to, or null when no guard did. + /// + /// The first value that no section read so far places, which the sections a whole read reads before make the file's + /// fault; otherwise the first a guard cut, the values past it read on — but for what depends on it: the /Size + /// an /Index so cut may stand beside, and the filters' values past one (#263). + /// private string? UnreadValue(PdfDictionary dictionary, out PdfObjectId? cut) { - if (Unread(dictionary, PdfName.W, out var widths, out cut) is { } fault) + var first = default(FirstUnread); + + if (first.Ends(Unread(dictionary, PdfName.W, out var widths, out cut), cut)) { - return fault; + return first.Fault; } for (var i = 0; widths is PdfArray w && i < Math.Min(3, w.Count); i++) { - if (Follow(PdfName.W, w[i], entry: false, out _, out cut) is { } elementFault) + if (first.Ends(Follow(PdfName.W, w[i], entry: false, out _, out cut), cut)) { - return elementFault; + return first.Fault; } } - if (Unread(dictionary, PdfName.Index, out var index, out cut) is { } indexFault) + var indexFault = Unread(dictionary, PdfName.Index, out var index, out cut); + + if (first.Ends(indexFault, cut)) { - return indexFault; + return first.Fault; } if (index is PdfArray ranges) { foreach (var element in ranges) { - if (Follow(PdfName.Index, element, entry: false, out _, out cut) is { } elementFault) + if (first.Ends(Follow(PdfName.Index, element, entry: false, out _, out cut), cut)) { - return elementFault; + return first.Fault; } } } - else if ((index is null or PdfNull) && Unread(dictionary, PdfName.Size, out _, out cut) is { } sizeFault) + else if (indexFault is null && (index is null or PdfNull) && first.Ends(Unread(dictionary, PdfName.Size, out _, out cut), cut)) { - return sizeFault; + return first.Fault; } var following = new Following(this); - var filtersRead = Filters.FilterChain.TryFollow(dictionary, _guard.Bound(PdfLimit.FilterChain), ref following); - cut = following.Cut; - return filtersRead ? null : following.Fault; + + if (!Filters.FilterChain.TryFollow(dictionary, _guard.Bound(PdfLimit.FilterChain), ref following)) + { + first.Ends(following.Fault, following.Cut); + } + + cut = first.Cut; + return first.Fault; + } + + /// + /// Keeps the first value of a cross-reference stream's dictionary a guard cut, until one that no section read so far + /// places, which replaces it. + /// + private struct FirstUnread + { + /// Gets what could not be read. + public string? Fault { get; private set; } + + /// Gets the object a guard cut that the value which could not be read leads to, if a guard cut one. + public PdfObjectId? Cut { get; private set; } + + /// Takes what a value could not be read for, and says whether it ends the reading: no guard cut it. + /// What the value could not be read for, or null when it could be. + /// The object a guard cut that left it unread, or null when none did. + public bool Ends(string? fault, PdfObjectId? cut) + { + if (fault is null) + { + return false; + } + + if (cut is null || Fault is null) + { + Fault = fault; + Cut = cut; + } + + return cut is null; + } } /// @@ -4600,11 +4710,9 @@ public bool TryRead(PdfName key, PdfObject? written, bool entry, out PdfObject? return $"its /Index is {DescribeValue(written)}, not an array"; } - if (ranges.Count == 0 || ranges.Count % 2 != 0) + if (PairsFault(ranges) is { } pairs) { - return string.Create( - CultureInfo.InvariantCulture, - $"its /Index holds {ranges.Count} {(ranges.Count == 1 ? "value" : "values")}, not pairs of a first object number and a count of rows"); + return pairs; } var checkedSubsections = new long[ranges.Count]; @@ -4614,33 +4722,55 @@ public bool TryRead(PdfName key, PdfObject? written, bool entry, out PdfObject? var start = ranges.Resolved(i).AsInteger(); var count = ranges.Resolved(i + 1).AsInteger(); - if (start is null || count is null) - { - return $"its /Index holds {DescribeValue(ranges[start is null ? i : i + 1])} where an integer belongs"; - } - - if (count < 0) + if (SubsectionFault(ranges, i, start, count) is { } fault) { - return "its /Index gives a subsection a count of rows out of range"; + return fault; } - // The last number is compared by subtraction: a count as large as a long would carry start + count - 1 past what - // one holds, and the rows past int.MaxValue would be read under negative numbers. - if (start is < 0 or > PdfObjectId.MaxNumber || count > PdfObjectId.MaxNumber - start + 1) - { - return string.Create( - CultureInfo.InvariantCulture, - $"its /Index gives a subsection of {count:N0} {(count == 1 ? "row" : "rows")} from object {start}, outside object numbers 0 to {PdfObjectId.MaxNumber}"); - } - - checkedSubsections[i] = start.Value; - checkedSubsections[i + 1] = count.Value; + checkedSubsections[i] = start.GetValueOrDefault(); + checkedSubsections[i + 1] = count.GetValueOrDefault(); } subsections = checkedSubsections; return null; } + /// Says what is wrong with how many values an /Index holds, or null when it holds pairs. + private static string? PairsFault(PdfArray ranges) => ranges.Count == 0 || ranges.Count % 2 != 0 + ? string.Create( + CultureInfo.InvariantCulture, + $"its /Index holds {ranges.Count} {(ranges.Count == 1 ? "value" : "values")}, not pairs of a first object number and a count of rows") + : null; + + /// + /// Says what is wrong with the subsection an /Index gives at , or null when nothing is: its + /// first number is an object number, and its count numbers object numbers alone. + /// + /// The /Index, whose written elements a fault describes. + /// Where the subsection's first number lies. + /// The first number, when it is an integer. + /// The count, when it is an integer. + private static string? SubsectionFault(PdfArray ranges, int at, long? start, long? count) + { + if (start is null || count is null) + { + return $"its /Index holds {DescribeValue(ranges[start is null ? at : at + 1])} where an integer belongs"; + } + + if (count < 0) + { + return "its /Index gives a subsection a count of rows out of range"; + } + + // The last number is compared by subtraction: a count as large as a long would carry start + count - 1 past what one + // holds, and the rows past int.MaxValue would be read under negative numbers. + return start is < 0 or > PdfObjectId.MaxNumber || count > PdfObjectId.MaxNumber - start + 1 + ? string.Create( + CultureInfo.InvariantCulture, + $"its /Index gives a subsection of {count:N0} {(count == 1 ? "row" : "rows")} from object {start}, outside object numbers 0 to {PdfObjectId.MaxNumber}") + : null; + } + /// /// Reads the offset a section's /Prev or /XRefStm gives, or -1 when it gives none. /// diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index a11966f6..29f40d97 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -2385,6 +2385,125 @@ public void A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_n document.WasRepaired.Should().BeFalse(); } + [Theory] + [InlineData("/Type /XRef /Size 8 /W [1 4] /Filter 5 0 R", "Filter", null)] + [InlineData("/Type /XRef /Size 8 /W [1 -4 2] /Filter 5 0 R", "Filter", null)] + [InlineData("/Type /XRef /Size 8 /W [0 0 0] /Filter 5 0 R", "Filter", null)] + [InlineData("/Type /XRef /Size 8 /W [1 /x 2] /Filter 5 0 R", "Filter", null)] + [InlineData("/Type /XRef /W [1 4] /Filter 5 0 R", "Filter", null)] + [InlineData("/Type /XRef /Size 9 0 R /W [1 4 2] /Filter 5 0 R", "Filter", "-1")] + [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [0 9 0 R] /Filter 5 0 R", "Filter", "-1")] + [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index 9 0 R /Filter 5 0 R", "Filter", "[0 8 3]")] + [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [0 5 0 R 3]", "Index element", null)] + [InlineData("/Type /XRef /Size 8 /W 5 0 R /Filter 9 0 R", "W", null)] + [InlineData("/Type /XRef /Size 8 /W 5 0 R /Index 9 0 R", "W", null)] + [InlineData("/Type /XRef /Size 8 /W [1 4] /Length 5 0 R", "Length", null)] + [InlineData("/Type /XRef /W [1 4 2] /Index [0] /Length 5 0 R", "Length", null)] + [InlineData("/Type /XRef /Size 8 /W 9 0 R /Length 5 0 R", "Length", null)] + public void What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_guard_cut_holds_is_the_file_s( + string entries, string cut, string? nine) + { + // Stream 7's /Filter, an /Index element, its /W or its /Length names object 5, which MaxObjectLength cuts — the length + // where no endstream ends the data. Its /W gives no three widths, or none past 0; its /Index holds no pairs, or a + // count of rows below 0, and its /Size no count, written direct or through object 9; or a value past the cut names + // object 9, which nothing places: no value the guard hid makes the section readable, and read whole it is the + // file's fault. Only a numbering written direct was judged: the section was the guard's, the chain ended, the index + // was not rebuilt and the catalog was lost (#263). + var (five, compressed) = cut switch + { + "Filter" => ("/FlateDecode" + new string(' ', 1100), true), + "Index element" => ("8" + new string(' ', 1100), false), + "W" => ("[1 4 2]" + new string(' ', 1100), false), + _ => (new string(' ', 1100) + "56", false), + }; + var built = ChainFiles.UnderAnUpdate( + cut == "Length" ? entries : entries + " /Length {length}", + five, + compressed, + predicted: false, + placed: true, + placedToo: nine is null ? null : new Dictionary { [9] = nine }); + var text = Encoding.Latin1.GetString(built); + + if (cut == "Length") + { + var endStream = text.IndexOf("endstream", text.IndexOf("7 0 obj", StringComparison.Ordinal), StringComparison.Ordinal); + text = text[..endStream] + "endstreaX" + text[(endStream + "endstream".Length)..]; + } + + var file = Encoding.Latin1.GetBytes(text); + using var whole = PdfDocument.Open(file); + using var document = PdfDocument.Open(file, ObjectGuard); + + var unreadable = whole.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.XRefSectionUnreadable).Which; + document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.XRefSectionUnreadable) + .Which.Message.Should().Be(unreadable.Message); + document.WasRepaired.Should().BeTrue(); + document.Reader.Structure.Sections.Should().NotContain(section => section.CutByLimit); + document.GetObject(new PdfObjectId(1)).AsDictionary().IsOfType(PdfName.Catalog).Should().BeTrue(); + } + + [Theory] + [InlineData("/W [1 4] /Size 8 /Filter 5 0 R")] + [InlineData("/W 5 0 R /Size 8 /Filter 9 0 R")] + public void A_place_a_section_may_have_moved_to_that_a_value_past_a_guard_s_cut_makes_unreadable_is_dropped(string entries) + { + // The /Prev lands 21 bytes into object 6, a stream whose /Filter, or /W, names object 5, which MaxObjectLength cuts, + // beside a /W that gives no three widths, or a /Filter that names object 9, which nothing places: no section, + // whatever the guard hid, as read whole. It was taken for the section the guard cut, and stream 7, the section, was + // lost (#263). + var built = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] /Length {length}", + (entries.StartsWith("/W 5", StringComparison.Ordinal) ? "[1 4 2]" : "/FlateDecode") + new string(' ', 1100), + compressed: false, + predicted: false, + placed: true, + six: "<< " + entries + " /Length 3 >>\nstream\nabc\nendstream"); + var text = Encoding.Latin1.GetString(built); + var stream = text.IndexOf("\n7 0 obj", StringComparison.Ordinal) + 1; + var six = text.IndexOf("\n6 0 obj", StringComparison.Ordinal) + 1; + var file = Encoding.Latin1.GetBytes(text.Replace(Invariant($"/Prev {stream} "), Invariant($"/Prev {six + 21} "), StringComparison.Ordinal)); + + using var whole = PdfDocument.Open(file); + using var document = PdfDocument.Open(file, ObjectGuard); + + document.Diagnostics.Select(entry => entry.Message).Should().Equal(whole.Diagnostics.Select(entry => entry.Message)); + document.Reader.Structure.Sections[^1].Offset.Should().Be(stream); + document.GetObject(new PdfObjectId(1)).AsDictionary().IsOfType(PdfName.Catalog).Should().BeTrue(); + } + + [Theory] + [InlineData("<< /Type /Font /W 5 0 R /Size 8 /Length 3 >>\nstream\nabc\nendstream", "[1 4 2]")] + [InlineData("<< /Type /Font /W [1 4 2] /Size 8 /Length 5 0 R >>\nstream\nabc\nendstreaX", "3")] + public void A_stream_of_another_type_whose_value_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut( + string six, string five) + { + // The /Prev names object 6 exactly, a stream that declares /Type /Font, whose /W, or whose /Length where no endstream + // ends its data before object 8, names object 5, which MaxObjectLength cuts: the guard cut no section, the stream + // saying what it is, as when the guard cuts its dictionary. It was taken for the section the guard cut, the chain + // ended there, and the catalog was lost; the section is found farther on (#263). + var built = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] /Length {length}", + five == "3" ? new string(' ', 1100) + five : five + new string(' ', 1100), + compressed: false, + predicted: false, + placed: true, + six: six, + placedToo: new Dictionary { [8] = "(eight)" }); + var text = Encoding.Latin1.GetString(built); + var stream = text.IndexOf("\n7 0 obj", StringComparison.Ordinal) + 1; + var sixAt = text.IndexOf("\n6 0 obj", StringComparison.Ordinal) + 1; + var file = Encoding.Latin1.GetBytes(text.Replace(Invariant($"/Prev {stream} "), Invariant($"/Prev {sixAt} "), StringComparison.Ordinal)); + + using var document = PdfDocument.Open(file, ObjectGuard); + + document.Reader.Structure.Sections[^1].Offset.Should().Be(stream); + document.Reader.Structure.Sections.Should().NotContain(section => section.CutByLimit); + document.GetObject(new PdfObjectId(1)).AsDictionary().IsOfType(PdfName.Catalog).Should().BeTrue(); + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.XRefOffsetAdjusted); + } + [Fact] public void A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild() { From 4e074ef6a4247f78abaf59a66a63bea3be417cc9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 18:25:04 +0000 Subject: [PATCH 115/146] fix: place a cross-reference stream's header past however much white space precedes it A cross-reference stream said where its header lies from the 32 bytes the section's probe reads, though the object is read past any white space before it. Named 32 bytes or more before its header, the section proved no endstream the stream before it ends at, whose data, whole, was taken for the guard's under a /Length a guard hid; the validator said the offset lay 32 bytes before the header. The section now takes where its header lies from the reading of its object. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 2 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 14 +++++++-- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 31 +++++++++++++++++++ 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index 0d7d53dd..e6c1e9fe 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -355,7 +355,7 @@ Known gaps: | A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | -| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`), so that a filter that runs out there — a Flate checksum short of a carriage return, an end marker missing — says nothing of the file, while the data decodes as read whole: white space parts no ASCII85 or ASCIIHex group, and no group is dropped nor a predictor's row undone for it. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `An_ascii85_object_stream_that_lacks_its_end_marker_before_a_proven_endstream_decodes_as_read_whole`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | +| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`), so that a filter that runs out there — a Flate checksum short of a carriage return, an end marker missing — says nothing of the file, while the data decodes as read whole: white space parts no ASCII85 or ASCIIHex group, and no group is dropped nor a predictor's row undone for it. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `A_cross_reference_stream_named_before_white_space_proves_the_endstream_before_it_however_much`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `An_ascii85_object_stream_that_lacks_its_end_marker_before_a_proven_endstream_decodes_as_read_whole`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | | Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, about the file once more; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 3766989d..68a6c08c 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3582,7 +3582,7 @@ private void ReadTrailerPastRows(long from, XRefSectionRecord section) section.TrailerPosition = found; var guards = _guardsReached; - _ = TryParseNumberedAt(found + TrailerKeyword.Length, DirectObject, PdfLimit.Trailer, out var value, out _, out var syntaxFault); + _ = TryParseNumberedAt(found + TrailerKeyword.Length, DirectObject, PdfLimit.Trailer, out var value, out _, out _, out var syntaxFault); if (_guardsReached != guards) { @@ -3647,7 +3647,7 @@ private bool StartsToken(long position) // The trailer starts inside the window, so inside the file as long as the source's length holds; // where nothing can be parsed the value is null, and anything that is not a dictionary is no trailer. var guards = _guardsReached; - _ = TryParseNumberedAt(absolute + position, DirectObject, PdfLimit.Trailer, out var value, out _, out var syntaxFault); + _ = TryParseNumberedAt(absolute + position, DirectObject, PdfLimit.Trailer, out var value, out _, out _, out var syntaxFault); if (_guardsReached != guards) { @@ -3835,6 +3835,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect PdfLimit.Trailer, out var value, out var number, + out var header, out var syntaxFault, XRefWindow, candidate ? DeclaresOnlyCrossReferenceStream : null) || @@ -3861,6 +3862,9 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect var dictionary = value is PdfStream parsed ? parsed.Dictionary : (PdfDictionary)value; section.Kind = XRefSectionKind.Stream; section.StreamObjectNumber = number; + + // The probe that looked for the xref keyword says where the header lies only within its first bytes. + section.Padding = (int)(header - absolute); section.TrailerPosition = absolute; section.TypedXRef = dictionary.IsOfType(PdfName.XRef); @@ -5280,7 +5284,7 @@ private PdfObject LoadFromEntry(PdfObjectId id, XRefEntry entry) /// past its end. Returns false when nothing with that number is there. /// private bool TryParseObjectAt(int number, long offset, out PdfObject value, long recordedAt = RecordedAnywhere) => - TryParseNumberedAt(offset, number, PdfLimit.Object, out value, out _, out _, recordedAt: recordedAt); + TryParseNumberedAt(offset, number, PdfLimit.Object, out value, out _, out _, out _, recordedAt: recordedAt); /// /// Parses what starts at an exact offset — object , any object for @@ -5302,6 +5306,7 @@ private bool TryParseObjectAt(int number, long offset, out PdfObject value, long /// The guard that bounds the window. /// The object read. /// The number of the object found, even when refuses it. + /// Where in the file the object read starts, past the white space and comments before it. /// /// Whether the reading kept met a fault of the syntax, or may have, the pending diagnostics full: reported now or /// already, by an earlier reading of the same bytes. @@ -5323,6 +5328,7 @@ private bool TryParseNumberedAt( PdfLimit limit, out PdfObject value, out int foundNumber, + out long headerAt, out bool syntaxFault, int initialWindow = InitialObjectWindow, Func? keep = null, @@ -5330,6 +5336,7 @@ private bool TryParseNumberedAt( { value = PdfNull.Instance; foundNumber = 0; + headerAt = offset; syntaxFault = false; if (offset < 0 || offset >= _source.Length) @@ -5435,6 +5442,7 @@ private bool TryParseNumberedAt( RecordEndObj(noted ? number : DirectObject, foundNumber, parser.EndObj, cut, offset); RecordStreamLength(noted ? number : DirectObject, foundNumber, parsed, lengthFault); value = parsed; + headerAt = offset + parser.HeaderStart; return true; } } diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 29f40d97..20d38a4b 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1157,6 +1157,37 @@ public void A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); } + [Theory] + [InlineData(20)] + [InlineData(40)] + public void A_cross_reference_stream_named_before_white_space_proves_the_endstream_before_it_however_much(int padding) + { + // As above, with no row, startxref naming the white space before the cross-reference stream's header: the section + // said where the header lies only within the first 32 bytes its probe read, and past them nothing proved the + // endstream the stream's own. The validator said the offset lies 32 bytes before the header (#224). + var file = new HandWrittenFile(version: "1.5"); + var rows = new Dictionary + { + [1] = (1, file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), 0), + [2] = (1, file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), 0), + [3] = (1, file.Object(3, Page), 0), + [5] = (1, file.Object(5, new string(' ', 2000) + "5"), 0), + [4] = (1, file.Object(4, "<< /Length 5 0 R >>\nstream\nBT ET\nendstream"), 0), + }; + file.Raw(new string(' ', padding)); + var header = file.CrossReferenceStream(6, 6, rows, "/Root 1 0 R"); + var bytes = Encoding.Latin1.GetBytes(Encoding.Latin1.GetString(file.Bytes) + .Replace(Invariant($"startxref\n{header}\n"), Invariant($"startxref\n{header - padding}\n"), StringComparison.Ordinal)); + using var document = PdfDocument.Open(bytes, ObjectGuard); + + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + stream.Data.CutByGuard.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + new PdfValidator().Validate(document).Findings.Should().ContainSingle(finding => finding.RuleId == PdfValidationRuleIds.XRefOffsetImprecise) + .Which.Message.Should().Contain(Invariant($"{padding} bytes before the header")); + } + [Fact] public void The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed() { From 86d38f55fa2f8217db355c1c19d8b55ce5e24902 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 18:30:29 +0000 Subject: [PATCH 116/146] test: hold the round's proofs, hidden ends and kept guard reports Each of these held no test, and a mutant of each passed the suite: data that ends in white space right before a proven endstream, its end left to the guard; a stream left unsearched whose /Length a guard left out of the index; a cross-reference stream the chain read proving the endstream of an older one before it; and a decoding guard's report kept beside a decoding withdrawn, held once and counted as left out where the diagnostics had no room for it. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 4 +- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 118 ++++++++++++++++++ .../AdCodicem.Pdf.Tests/StreamLengthTests.cs | 29 +++-- 3 files changed, 138 insertions(+), 13 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index e6c1e9fe..27406411 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -284,7 +284,7 @@ Known gaps: | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). A hybrid file's stream's own `/Prev`, which the chain does not follow, going on through the table's, is not read at all, nor are the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place: neither names a section missing nor ends the chain for a guard, and the rule on trailer values judges them as written, a reference they hold not followed ([#263]). An object the guard cut after a whole reference it holds hides nothing: the chain reads through it, as a load does, and loads it once the chain is read. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s`, `A_table_an_xrefstm_names_in_a_stream_s_place_has_its_own_links_left_unread`, `A_prev_written_as_a_reference_to_a_real_is_judged_through_it_only_where_the_reader_follows_it` | Yes | | What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); so is one that declared itself a section before a guard cut it — a table's `xref` keyword, `/Type /XRef`. A place that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is a guess dropped, the next place tried; where the chain names a section, a guard that cut what declares another type — its dictionary or a value of it — cut no section, and relocation is tried as read whole. What makes a section unreadable whatever a value a guard cut holds is the file's fault, as read whole, judged before the cut is taken for the guard's (`PdfFileReader.FaultBesideCut`): a value past the cut that no section read so far places, which the sections a whole read reads before make the file's to lack; a `/W` that gives no three widths, or none past 0; rows numbered wrongly — an `/Index` that holds no pairs, whatever its elements, a subsection out of range, a `/Size` with no `/Index` that gives no count —, as far as the values read, written direct or through references, say: a value a guard cut says nothing, nor does what depends on it, the `/Size` an `/Index` so cut may stand beside | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_cross_reference_stream_whose_deferred_length_a_guard_cut_is_found_where_it_was_moved`, `A_cross_reference_stream_a_guard_cut_the_dictionary_of_after_its_type_is_the_section_where_it_was_moved`, `A_classic_table_a_guard_cut_the_rows_of_is_the_section_where_it_was_moved`, `A_stream_with_no_numbering_whose_width_a_guard_cut_is_no_section_the_guard_cut`, `A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild`, `What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_guard_cut_holds_is_the_file_s`, `A_place_a_section_may_have_moved_to_that_a_value_past_a_guard_s_cut_makes_unreadable_is_dropped`, `A_stream_of_another_type_whose_value_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, may stay within the bound — the group counted against what the filters before the last share as well —: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `Data_whose_end_a_guard_hides_leaves_the_filters_after_a_dropped_group_the_room_its_direct_twin_s_have`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, may stay within the bound — the group counted against what the filters before the last share as well —: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_decoding_guard_deferred_data_reached_is_reported_once_whatever_room_the_diagnostics_have`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `Data_whose_end_a_guard_hides_leaves_the_filters_after_a_dropped_group_the_room_its_direct_twin_s_have`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file`, `A_cross_reference_stream_the_chain_read_proves_the_endstream_of_the_older_one_before_it` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, written direct — an integer, or a real with no fractional part; one written as a reference is not read through for it, and bounds nothing —, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | @@ -355,7 +355,7 @@ Known gaps: | A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | -| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`), so that a filter that runs out there — a Flate checksum short of a carriage return, an end marker missing — says nothing of the file, while the data decodes as read whole: white space parts no ASCII85 or ASCIIHex group, and no group is dropped nor a predictor's row undone for it. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `A_cross_reference_stream_named_before_white_space_proves_the_endstream_before_it_however_much`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `An_ascii85_object_stream_that_lacks_its_end_marker_before_a_proven_endstream_decodes_as_read_whole`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | +| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`), so that a filter that runs out there — a Flate checksum short of a carriage return, an end marker missing — says nothing of the file, while the data decodes as read whole: white space parts no ASCII85 or ASCIIHex group, and no group is dropped nor a predictor's row undone for it. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `A_cross_reference_stream_named_before_white_space_proves_the_endstream_before_it_however_much`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `Data_that_ends_in_white_space_right_before_a_proven_endstream_ends_where_the_guard_hides`, `An_ascii85_object_stream_that_lacks_its_end_marker_before_a_proven_endstream_decodes_as_read_whole`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | | Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, about the file once more; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 20d38a4b..8b0d38b3 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -899,6 +899,48 @@ public void Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_ends guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries); } + [Theory] + [InlineData("cut", true)] + [InlineData("left out", true)] + [InlineData("cut", false)] + public void Data_that_ends_in_white_space_right_before_a_proven_endstream_ends_where_the_guard_hides(string guard, bool space) + { + // Content stream 4's hexadecimal data lacks its > and ends in a space, the endstream right after it, which the next + // object proves its own: whether the space is the data's only the hidden length says, and the missing > is no fault + // the file shows where the data ends. With no space, the data ends where the file's does, and the > it lacks is the + // file's fault (#224). + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, Page), + [20] = file.Object(20, (guard == "cut" ? new string(' ', 2000) : string.Empty) + (space ? "11" : "10")), + [4] = file.Object(4, $"<< /Length 20 0 R /Filter /ASCIIHexDecode >>\nstream\n48656C6C6F{(space ? " " : string.Empty)}endstream"), + [5] = file.Object(5, "(five)"), + }; + file.Table(21, rows, "/Size 21 /Root 1 0 R"); + var limits = guard == "cut" + ? PdfReaderLimits.Default with { MaxObjectLength = 1024 } + : PdfReaderLimits.Default with { MaxIndexEntries = 20 }; + + using (var whole = PdfDocument.Open(file.Bytes)) + { + _ = whole.GetObject(new PdfObjectId(4)).AsStream().Required().Decode(); + whole.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.FilterFailed); + } + + using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = limits }); + var content = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + var reached = guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries; + + content.Decode().ToArray().Should().Equal("Hello"u8.ToArray()); + content.Data.CutByGuard.Should().BeFalse(); + content.Data.EndHidden.Should().Be(space); + document.Reader.IsEndObjMissing(4, out _).Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(space ? [reached] : [reached, PdfDiagnosticCodes.FilterFailed]); + } + [Theory] [InlineData("cut", "(world)", "world")] [InlineData("left out", "(world)", "world")] @@ -1130,6 +1172,34 @@ public void A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decod findings.Should().Contain(PdfValidationRuleIds.XRefCheckedInPart).And.NotContain(PdfValidationRuleIds.XRefSectionMalformed); } + [Theory] + [InlineData(1000, 1, 0)] + [InlineData(1, 0, 1)] + [InlineData(0, 0, 1)] + public void A_decoding_guard_deferred_data_reached_is_reported_once_whatever_room_the_diagnostics_have( + int capacity, int reports, int leftOut) + { + // As above, under 279: the guard the first decoding reached stands, its report held as the stream's decoding + // reports are, so that decoding the stream again reports it no more; where the diagnostics had no room for it, it + // is counted as left out, while the ASCIIHex end fault the first decoding met, dropped with that decoding, is not + // (#144, #220, #224). + var options = new PdfReaderOptions + { + Limits = PdfReaderLimits.Default with { MaxDecodedStreamLength = 279 }, + DiagnosticCapacity = capacity, + }; + using var document = PdfDocument.Open(ChainFiles.HexFlateRowsLackingTheirLastDigit(), options); + + document.Diagnostics.Count.Should().Be(reports); + document.Diagnostics.SuppressedCount.Should().Be(leftOut); + + var stream = document.GetObject(new PdfObjectId(5)).AsStream().Required(); + _ = stream.Decode(); + _ = stream.Decode(); + + document.Diagnostics.Count(entry => entry.Code == PdfDiagnosticCodes.LimitDecodedStream).Should().Be(reports); + } + [Theory] [InlineData(true)] [InlineData(false)] @@ -1157,6 +1227,54 @@ public void A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); } + [Fact] + public void A_cross_reference_stream_the_chain_read_proves_the_endstream_of_the_older_one_before_it() + { + // Older cross-reference stream 6 takes its /Length from object 9, which MaxObjectLength cuts, and declares 11 rows + // where its data holds 10; newer stream 7, which no row of its own places, follows its endobj, the only startxref + // after it. The chain read stream 7 first, which proves the endstream before it the older stream's own: the row it + // lacks is the file's to lack, as read whole, not the guard's (#224). + var file = new HandWrittenFile(version: "1.5"); + var placed = new Dictionary + { + [1] = (1, file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), 0), + [2] = (1, file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), 0), + [3] = (1, file.Object(3, Page), 0), + [9] = (1, file.Object(9, new string(' ', 2000) + "70"), 0), + }; + var older = file.Position; + placed[6] = (1, older, 0); + file.Object(6, "<< /Type /XRef /Size 11 /W [1 4 2] /Root 1 0 R /Length 9 0 R >>\nstream\n" + Rows(10, placed) + "\nendstream"); + var newer = file.Position; + var rows = Rows(10, placed); + file.Object(7, Invariant($"<< /Type /XRef /Size 10 /W [1 4 2] /Root 1 0 R /Prev {older} /Length {rows.Length} >>\nstream\n{rows}\nendstream")); + file.Raw(Invariant($"startxref\n{newer}\n%%EOF\n")); + + using var whole = PdfDocument.Open(file.Bytes); + using var document = PdfDocument.Open(file.Bytes, ObjectGuard); + var section = document.Reader.Structure.Sections[^1]; + + section.Incomplete.Should().BeTrue(); + section.CutByLimit.Should().BeFalse(); + section.Fault.Should().Be(whole.Reader.Structure.Sections[^1].Fault).And.Be("it holds 10 rows where its /Index and /Size declare 11"); + + static string Rows(int size, Dictionary placed) + { + var data = new StringBuilder(); + + for (var row = 0; row < size; row++) + { + var (type, field2, field3) = row == 0 ? (0, 0L, 65535) : placed.TryGetValue(row, out var given) ? given : (0, 0L, 0); + data.Append((char)type) + .Append((char)((field2 >> 24) & 0xFF)).Append((char)((field2 >> 16) & 0xFF)) + .Append((char)((field2 >> 8) & 0xFF)).Append((char)(field2 & 0xFF)) + .Append((char)((field3 >> 8) & 0xFF)).Append((char)(field3 & 0xFF)); + } + + return data.ToString(); + } + } + [Theory] [InlineData(20)] [InlineData(40)] diff --git a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs index 041a19ac..702ee8a7 100644 --- a/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs +++ b/tests/AdCodicem.Pdf.Tests/StreamLengthTests.cs @@ -2231,21 +2231,27 @@ public void An_endstream_the_data_spells_past_which_the_searches_may_read_no_mor document.Reader.IsEndObjMissing(Streams + 3, out _).Should().BeFalse(); } - [Fact] - public void A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s() - { - // As above, stream 13's data running 3,000 bytes past the bytes its object is parsed with: once the searches have - // read as much as they may, it is not searched past them, and where it ends is unknown — the length that would say - // is the guard's. The data up to the next object, its endstream and endobj taken in, was the file's, reported as a - // length the reader did not search past; it is the guard's, as an endstream found and not proven is (#224). + [Theory] + [InlineData("cut")] + [InlineData("left out")] + public void A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s(string guard) + { + // As above, stream 13's data running 20,000 bytes past the bytes its object is parsed with, its /Length cut by + // MaxObjectLength or left out of the index by MaxIndexEntries: once the searches have read as much as they may, it is + // not searched past them, and where it ends is unknown — the length that would say is the guard's. The data up to the + // next object, its endstream and endobj taken in, or to the end of the file, was the file's, reported as a length the + // reader did not search past; it is the guard's, as an endstream found and not proven is (#224). const int Streams = 10; - var data = "BT " + new string('x', 3000) + " ET"; + var data = "BT " + new string('x', 20_000) + " ET"; var (file, _) = NestedStreams( Streams, 100_000, Invariant($"{Streams + 3} 0 obj\n<< /Length {Streams + 4} 0 R >>\nstream\n{data}\nendstream\nendobj\n"), - Invariant($"{Streams + 4} 0 obj\n{new string(' ', 2000)}{data.Length}\nendobj\n")); - using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxObjectLength = 1024 } }); + Invariant($"{Streams + 4} 0 obj\n{(guard == "cut" ? new string(' ', 2000) : string.Empty)}{data.Length}\nendobj\n")); + var limits = guard == "cut" + ? PdfReaderLimits.Default with { MaxObjectLength = 1024 } + : PdfReaderLimits.Default with { MaxIndexEntries = Streams + 4 }; + using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = limits }); for (var number = 3; number <= Streams + 2; number++) { @@ -2257,7 +2263,8 @@ public void A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s() stream.Data.CutByGuard.Should().BeTrue(); document.Reader.IsEndObjMissing(Streams + 3, out _).Should().BeFalse(); - document.Diagnostics.Skip(reports).Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.Diagnostics.Skip(reports).Select(entry => entry.Code).Should().Equal( + guard == "cut" ? [PdfDiagnosticCodes.LimitObject] : []); } [Fact] From ae25a0aa6995b6b4994ae0c0287901455c5981c8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 18:41:23 +0000 Subject: [PATCH 117/146] fix: read the white space an endstream's proof crosses once more at most The proof that an endstream found under a hidden /Length is the stream's own read again the white space before the endobj that the read-on after the keyword had read, and doubled through the white space a row names before the next header before reading it once more from the row. Thirty streams followed by 64 KB of white space had the searches read close to four times the file, the most they may, where three were said. The proof takes where the endobj lies from the read-on, and reads from a row's place in one first read that reaches the header weighed: the same files read 2 to 2.9 times their length. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- ...-is-readable-and-the-readers-guards-are.md | 4 +- docs/threat-model.md | 2 +- docs/website/docs/concepts/diagnostics.md | 3 +- docs/website/docs/reference/reader-limits.md | 2 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 50 +++++++++++++------ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 43 ++++++++++++++++ 6 files changed, 84 insertions(+), 20 deletions(-) diff --git a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md index ac74c12f..e5f28c65 100644 --- a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md +++ b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md @@ -101,7 +101,9 @@ constant with its reason rewritten: a valid file is read past the parser's windo cross-reference stream whose `/Length` the chain cannot read, by the search for the data of a stream whose `/Length` a guard cut or left out of the index, and by white space and comments after a declared length that run past a first read — about twice the file at most; the proof that an `endstream` found under a hidden `/Length` is the stream's -own reads 48 or 64 bytes past each, once, about the file once more, three times it in all (amended on 2026-10-24). +own reads 48 or 64 bytes past each, once, and the white space it crosses once more at most — the run before an +`endobj` taken from the read-on that read it, a run a row names before the next header read once from the row —, about +three times the file in all (amended on 2026-10-24). `MaxEndStreamGap`, the four bytes of white space accepted between a stream's data and its `endstream`, is removed: more, or a comment, made a right `/Length` a wrong one, and white space and comments are read on as far as they run, counted with the searches — a comment only once white space diff --git a/docs/threat-model.md b/docs/threat-model.md index 27406411..64859ab4 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -356,7 +356,7 @@ Known gaps: | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | | A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`), so that a filter that runs out there — a Flate checksum short of a carriage return, an end marker missing — says nothing of the file, while the data decodes as read whole: white space parts no ASCII85 or ASCIIHex group, and no group is dropped nor a predictor's row undone for it. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `A_cross_reference_stream_named_before_white_space_proves_the_endstream_before_it_however_much`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `Data_that_ends_in_white_space_right_before_a_proven_endstream_ends_where_the_guard_hides`, `An_ascii85_object_stream_that_lacks_its_end_marker_before_a_proven_endstream_decodes_as_read_whole`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | -| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, about the file once more; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | +| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, and the white space it crosses once more at most — the run before an `endobj` taken from the read-on that read it, a run a row names before the next header read once from the row —, about three times the file in all; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_proofs_of_streams_white_space_follows_read_no_stretch_of_the_file_more_than_three_times`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | | Encoded data of any length: a `/Length` of a few digits claiming gigabytes, one past what an array holds, which would exhaust memory at each read, or data decoding hands back as it is, which `MaxDecodedStreamLength` does not bound | `MaxStreamLength` (*The guards*): what is read of one stream's data at once stops at it, what lies past it is cut by a guard and reported as `limit.stream-length` — once for the stream in its document's diagnostics, and with each decoding that meets the cut in diagnostics a caller passes to `Decode`, as `limit.decoded-stream` is —, and the filters decode what was read as data the reader cut ([#170]) | `ReaderLimitsTests.A_stream_cut_at_MaxStreamLength_decodes_what_lies_within_it_as_data_the_reader_cut_and_is_reported_once`, `Data_decoding_hands_back_as_it_is_is_bounded_by_MaxStreamLength_and_not_by_MaxDecodedStreamLength`, `A_length_past_what_an_array_holds_ends_at_the_guard_and_never_in_an_exhausted_memory`, `Decoding_into_diagnostics_of_one_s_own_reports_the_cut_max_stream_length_made_each_time` | No | diff --git a/docs/website/docs/concepts/diagnostics.md b/docs/website/docs/concepts/diagnostics.md index 299b7b24..b2fc3fbe 100644 --- a/docs/website/docs/concepts/diagnostics.md +++ b/docs/website/docs/concepts/diagnostics.md @@ -101,7 +101,8 @@ what was read first. A file crafted for it can still make what a stream holds de first. What the searches of one document read together is bounded: none starts once they have read four times the file's length. For a valid file, what they read — its cross-reference streams searched, its streams whose length a guard hid, the white space after a declared length read on — comes to about twice the file at most, and the proofs that an `endstream` -found under a hidden length is the stream's own, 48 or 64 bytes past each, once, to about the file once more; the searches of damaged +found under a hidden length is the stream's own, 48 or 64 bytes past each, once, and the white space they cross once more at +most, to about three times the file in all; the searches of damaged streams read twice the file at most — 0.35 of it at most, all searches together, in the test corpus — unless their stretches overlap: one object's header inside another's dictionary, or a header the search does not take — a regular character glued before its number — that the diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index 0c944da9..98315d51 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -199,7 +199,7 @@ the documents producers write. They are not options: |---|---|---| | Nesting of arrays and dictionaries | 128 levels | Real documents nest a handful of levels; the bound keeps a hostile file from exhausting the stack | | A reference that resolves to another reference | followed 32 times | Producers write the object itself; the bound breaks a chain that loops | -| Searches for a stream's `endstream` | four times the file's length, for a document's searches together | A valid stream's `endstream` follows its declared length: the file is read past the bytes an object is parsed through only for the data of a cross-reference stream whose `/Length` the chain cannot read, for the data of a stream whose `/Length` a guard cut or left out of the index, and for white space and comments after a declared length that run past a first read — about twice the file at most —, and the proof that an `endstream` found under a hidden `/Length` is the stream's own reads 48 or 64 bytes past each, once, about the file once more. Past the bound, a stream whose `endstream` the bytes read with its object do not show is not searched for further, and its report says so — or, where a guard hid its `/Length`, its data up to the next object is the guard's, `CutByGuard`, nothing reported | +| Searches for a stream's `endstream` | four times the file's length, for a document's searches together | A valid stream's `endstream` follows its declared length: the file is read past the bytes an object is parsed through only for the data of a cross-reference stream whose `/Length` the chain cannot read, for the data of a stream whose `/Length` a guard cut or left out of the index, and for white space and comments after a declared length that run past a first read — about twice the file at most —, and the proof that an `endstream` found under a hidden `/Length` is the stream's own reads 48 or 64 bytes past each, once, and the white space it crosses once more at most, about three times the file in all. Past the bound, a stream whose `endstream` the bytes read with its object do not show is not searched for further, and its report says so — or, where a guard hid its `/Length`, its data up to the next object is the guard's, `CutByGuard`, nothing reported | | Objects loaded while loading another | 64 | An indirect `/Length` leads to an integer, and a stream resolves its filters and the parameters they read before decoding any of its data: no level holds decoded data while the next loads | | Object numbers | up to 2,147,483,647 | An object's number is an `int`, and ISO 32000-1's Annex C advises at most 8,388,607 indirect objects. A number past it is refused where it is read — a header, a reference, a subsection, a row, an object stream's header —, never narrowed to one that would name another object. An object is numbered from 1; 0 names the head of the free list, where a reference, a subsection or a row may give it | | An object stream's members | each read to the next one's start, the last to the end of the decoded data | ISO 32000-1 (7.5.7) has a header list its objects by increasing offset, so that each lies before the next one starts: a pair whose offset is not past the one before it, or whose object would start past the data, ends the header there, as a stray token does, and the objects listed before it are read. A member past `MaxObjectLength` is that limit's | diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 68a6c08c..b7e99ca9 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -154,20 +154,22 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// fourth adds a little to each stream of the second way: the proof that an endstream found is the stream's /// own (#224) reads past it to its object's end and what follows, three first reads of /// bytes, and a fourth past a row that names the white space before the next object's header, each once however - /// often the stream is parsed, doubling only as white space runs on — 48 or 64 bytes, about what the smallest such - /// stream spans with its header, so that the proofs add the file once more at most: three times it in all, within - /// the four the searches may read. Every other search is of an invalid stream, whose endstream does not - /// follow its declared length. Each reads from a stream's data to the first object header the bytes hold after it, - /// or to the next object the index as written places, whichever is nearer, and its reads grow from twice what the - /// parser holds of the data (): it reads at most about twice what its object - /// spans, and the searches of objects whose headers follow one another read disjoint stretches — the file twice - /// over at most. Only searches that share a stretch read more: objects that overlap, the header of one inside the - /// dictionary of another, before its data; or a stream whose header the search does not take — a regular character - /// glued before its number, where a rebuild's scan takes the digits after it for one — and the index as written - /// does not place, which only a rebuilt one does. This keeps what they read in proportion to the file, as a window - /// keeps the parsing of their dictionaries. Which streams keep their declared length unsearched past it depends on - /// the order they are read in: crafted files reach it, and no document of the corpus comes near, whose searches - /// read 0.35 of a file at most. + /// often the stream is parsed — 48 or 64 bytes, about what the smallest such stream spans with its header. White + /// space it crosses adds to that: the run before the object's endobj it takes from the read-on that read it, + /// the run after it it doubles through, as the read-on does, and the run a row names before the next header it + /// reads once from there, a first read reaching the header. No run is read more than three times, and the file + /// about three times at most in all, within the four the searches may read. Every other search is of an invalid + /// stream, whose endstream does not follow its declared length. Each reads from a stream's data to the first + /// object header the bytes hold after it, or to the next object the index as written places, whichever is nearer, + /// and its reads grow from twice what the parser holds of the data (): it reads + /// at most about twice what its object spans, and the searches of objects whose headers follow one another read + /// disjoint stretches — the file twice over at most. Only searches that share a stretch read more: objects that + /// overlap, the header of one inside the dictionary of another, before its data; or a stream whose header the + /// search does not take — a regular character glued before its number, where a rebuild's scan takes the digits + /// after it for one — and the index as written does not place, which only a rebuilt one does. This keeps what they + /// read in proportion to the file, as a window keeps the parsing of their dictionaries. Which streams keep their + /// declared length unsearched past it depends on the order they are read in: crafted files reach it, and no + /// document of the corpus comes near, whose searches read 0.35 of a file at most. /// internal const int EndStreamSearchPasses = 4; @@ -401,6 +403,13 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// private readonly Dictionary _endStreamChecks = []; + /// + /// Where the first significant byte lies past each endstream keyword the reader read on past (#174), which an + /// endobj there starts at: the proof that the keyword ends the stream's own data (#224) takes it rather than read + /// the white space again. + /// + private readonly Dictionary _endObjStarts = []; + /// /// What follows the object's end after each place a search ended data at, with what was said follows the /// endstream there, as reads it: where its first significant @@ -1722,6 +1731,12 @@ long IPdfStreamDataProvider.DataStart(long absoluteKeywordEnd) var next = NextSignificant(keywordEnd); state = next < 0 ? EndObjState.Unseen : IsEndObjKeywordAt(next) ? EndObjState.Present : EndObjState.Absent; _endStreamChecks[absoluteOffset] = state; + + if (state == EndObjState.Present) + { + _endObjStarts[keywordEnd] = next; + } + return state; } @@ -2012,7 +2027,7 @@ private bool IsStreamSectionAt(long position) if (end >= 0 && endObj == EndObjState.Present) { - end = NextSignificant(end, ProofRead); + end = _endObjStarts.TryGetValue(end, out var endObjAt) ? endObjAt : NextSignificant(end, ProofRead); end = end < 0 || end >= _source.Length ? -1 : end + StreamEnds.EndObjKeyword.Length; } @@ -2042,9 +2057,10 @@ private bool IsPlacedAt(long position, long? placed) return true; } + // The first read reaches the place weighed: white space before it is read once, not doubled through. if (!_placedStarts.TryGetValue(at, out var start)) { - start = NextSignificant(at, ProofRead); + start = NextSignificant(at, (int)Math.Clamp(position - at + 1, ProofRead, EndStreamSearchChunk)); _placedStarts[at] = start; } @@ -2317,6 +2333,8 @@ internal void Close() _endStreamSearches.TrimExcess(); _endStreamChecks.Clear(); _endStreamChecks.TrimExcess(); + _endObjStarts.Clear(); + _endObjStarts.TrimExcess(); _endStreamFollowers.Clear(); _endStreamFollowers.TrimExcess(); _placedStarts.Clear(); diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 8b0d38b3..04d1fedf 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1387,6 +1387,49 @@ public void The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_ document.Reader.EndStreamSearchBytes.Should().Be(Streams * 48, "each proof reads three times 16 bytes").And.BeLessThan(bytes.Length); } + [Theory] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public void The_proofs_of_streams_white_space_follows_read_no_stretch_of_the_file_more_than_three_times( + bool beforeEndObj, bool beforeNextHeader) + { + // Thirty streams whose /Length, object 20, MaxObjectLength cuts end in 65,520 bytes of white space, before their endobj + // or before the next object's header, which its row names the start of. The proof read again the white space before + // the endobj the reader had read on through, and doubled through that before the next header before reading it once + // more from the row: close to four times the file, the most the searches may read, where three were said (#224). + const int Streams = 30; + var run = new string(' ', 65_520); + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, Page), + [20] = file.Object(20, new string(' ', 1100) + "1"), + }; + + for (var number = 30; number < 30 + Streams; number++) + { + rows[number] = file.Position; + file.Raw(Invariant( + $"{(beforeNextHeader ? run : string.Empty)}{number} 0 obj\n<< /Length 20 0 R >>\nstream\nq\nendstream{(beforeEndObj ? run : string.Empty)}\nendobj\n")); + } + + rows[30 + Streams] = file.Object(30 + Streams, "(end)"); + file.Table(31 + Streams, rows, Invariant($"/Size {31 + Streams} /Root 1 0 R")); + var bytes = file.Bytes; + + using var document = PdfDocument.Open(bytes, ObjectGuard); + + for (var number = 30; number < 30 + Streams; number++) + { + document.GetObject(new PdfObjectId(number)).AsStream().Required().Data.CutByGuard.Should().BeFalse(); + } + + document.Reader.EndStreamSearchBytes.Should().BeLessThan(3L * bytes.Length); + } + [Theory] [InlineData("Rx", false)] [InlineData("RG", false)] From 936977677a603906b9a4c21d34bce4702a9fe647 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 18:45:50 +0000 Subject: [PATCH 118/146] docs: say where a guard's cut leaves the chain going on and a hidden end silent The reference said limit.trailer ends the chain where the dictionary cut declared another type, and that a /Prev or /XRefStm leading to a cut object ends it, though a hybrid stream's own /Prev and the links of a table an /XRefStm names are not read; that a deferred length is not checked on data a guard cut that holds every row; that a section a value's cut keeps from being read is the guard's, with no word of the faults the values read still charge to the file; and an endobj's proof without the cross-reference stream the chain read. The cost of a hidden end, end faults a whole read reports left silent, is recorded as an accepted gap, and three summaries corrected. Refs #224, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/architecture.md | 4 ++-- docs/threat-model.md | 5 +++++ docs/website/docs/concepts/validation.md | 4 ++-- docs/website/docs/reference/diagnostics.md | 14 +++++++++----- docs/website/docs/reference/validation-rules.md | 8 ++++---- src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs | 4 +++- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 10 +++++----- src/AdCodicem.Pdf/IO/PdfObjectParser.cs | 5 +++-- src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs | 3 ++- 9 files changed, 35 insertions(+), 22 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 48d8d922..ec641916 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -135,8 +135,8 @@ separate types, separate severity scales, and vocabularies that never share a co option, on by default and reported under its own `limit.*` code when reached; a bound only an invalid file reaches is an internal constant (ADR 34). What a guard cut is the guard's, never the file's fault: a value that leads to an object a guard cut or left out of the index — a stream's `/Length`, `/Filter` or - `/DecodeParms`, a cross-reference stream's `/W`, a trailer's `/Prev` or `/XRefStm` — is not judged as one the - file gave. The stream's data is found by its `endstream`, an integer the cut left only a place to look for + `/DecodeParms`, a cross-reference stream's `/W`, a trailer's `/Prev` or `/XRefStm` the reader follows — is not + judged as one the file gave. The stream's data is found by its `endstream`, an integer the cut left only a place to look for one at, and a missing `endstream` is still the file's fault but where the search stops at a header the data spells or an `endstream` nothing after it proves the stream's own (item 4); a reference the cut left whole is read through, as a load reads it; its chain of filters ends there, with the data diff --git a/docs/threat-model.md b/docs/threat-model.md index 64859ab4..d8741a00 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -380,6 +380,11 @@ Known gaps: - Accepted, with no issue: under a `/Length` a guard cut or left out of the index, data that spells `endstream`, an `endobj` or not, then `xref` or `startxref`, and goes on, ends there as the file's data, and its object as ending there: only the hidden length could tell, and a valid file's last stream before its table is told so ([#224]). +- Accepted, with no issue: under a `/Length` a guard cut or left out of the index, data an `endstream` proven its own + ends past white space — the end-of-line the search leaves out before the keyword, above all — decodes with its end + the guard's: a Flate stream that lost its tail or its checksum, or ASCII85, ASCIIHex, RunLength or LZW data without + its end marker, is not reported there, though a whole read reports it; only the hidden length says whether that white + space is the data's ([#224]). ### Decoding diff --git a/docs/website/docs/concepts/validation.md b/docs/website/docs/concepts/validation.md index 7e8dbc79..21e0ada8 100644 --- a/docs/website/docs/concepts/validation.md +++ b/docs/website/docs/concepts/validation.md @@ -156,8 +156,8 @@ limit is not a fault of the file, so a rule meeting one judges nothing of what i information, that it could not check it whole, rather than calling the file broken; opening the document with a raised limit lets it check the rest. The same holds of a value that leads to such an object: a cross-reference stream whose `/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, leads to one is said not checked rather than -malformed — so is one -whose `/Length`, which the chain could not read, leads to one where only that length could say where its data ends —; +malformed — so is one whose `/Length`, which the chain could not read, leads to one where only that length could say +where its data ends —, unless what the values read say makes it unreadable whatever the guard hid, which is the file's; a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain there rather than naming a section not found; and an object stream whose `/Filter` or `/DecodeParms` leads to one, or to an object a limit left out of the index, is left encoded, the objects it holds said not checked rather than lost diff --git a/docs/website/docs/reference/diagnostics.md b/docs/website/docs/reference/diagnostics.md index 4bba7191..f8fc0722 100644 --- a/docs/website/docs/reference/diagnostics.md +++ b/docs/website/docs/reference/diagnostics.md @@ -101,11 +101,11 @@ Codes are stable: they are part of the public contract, because callers filter o | `header.missing` | The file has no `%PDF-` header in its first 4,096 bytes; it was read all the same, its offsets counted from its first byte | | `xref.rebuilt` | The index was rebuilt by scanning the whole file | | `xref.offset-adjusted` | An object, or a cross-reference section the chain names, was not where the file said, and was found nearby | -| `xref.section-missing` | A cross-reference section `/Prev` or `/XRefStm` names holds nothing that reads as a section, nor does any place nearby, lies outside the file, or is named by a value that is not an offset; what only it indexed is found by rebuilding the index when it is asked for. Reported at the offset the section was named at, or, when that offset lies outside the file or the value is not an offset, at the section whose trailer named it. A `/Prev` or `/XRefStm` written as a reference that leads to an object one of the reader's limits cut names what the guard hides: no section is reported missing, nothing is rebuilt, the chain ends there, and the guard is reported once the chain is read | -| `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, data or — with no `/Index` — `/Size` cannot give rows, or whose `/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, is a reference no section read before it places where it can be read without loading it, or whose `/Length` is such a reference, or leads to one through others, and no `endstream` follows its data before the next object the sections read so far place, or before the end of the file, or none was looked for once the document's searches for `endstream` read as much of the file as they may — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message. A value that names an object one of the reader's limits cut, directly or through a chain of references — a `/Length` whose data no `endstream` ends before that stop among them —, is the guard's, not the file's: the section is not reported, nor looked for nearby, the chain ends there, nothing is rebuilt, what only that section and the older ones index reads as null, and the guard is reported once the chain is read | +| `xref.section-missing` | A cross-reference section `/Prev` or `/XRefStm` names holds nothing that reads as a section, nor does any place nearby, lies outside the file, or is named by a value that is not an offset; what only it indexed is found by rebuilding the index when it is asked for. Reported at the offset the section was named at, or, when that offset lies outside the file or the value is not an offset, at the section whose trailer named it. A `/Prev` or `/XRefStm` the reader follows — not a hybrid file's cross-reference stream's own `/Prev`, nor the links of a table an `/XRefStm` names, which it leaves unread —, written as a reference that leads to an object one of the reader's limits cut, names what the guard hides: no section is reported missing, nothing is rebuilt, the chain ends there, and the guard is reported once the chain is read | +| `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, data or — with no `/Index` — `/Size` cannot give rows, or whose `/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, is a reference no section read before it places where it can be read without loading it, or whose `/Length` is such a reference, or leads to one through others, and no `endstream` follows its data before the next object the sections read so far place, or before the end of the file, or none was looked for once the document's searches for `endstream` read as much of the file as they may — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message. A value that names an object one of the reader's limits cut, directly or through a chain of references — a `/Length` whose data no `endstream` ends before that stop among them —, is the guard's, not the file's: the section is not reported, nor looked for nearby, the chain ends there, nothing is rebuilt, what only that section and the older ones index reads as null, and the guard is reported once the chain is read — but for what makes the section unreadable whatever the value holds, which is the file's and reported here, as read whole: a value past it no section read so far places, a `/W` that gives no three widths or none past 0, or rows numbered wrongly as far as the values read say; and an object that declares another type, a font say, is no section a guard cut a value of, and the section is looked for nearby | | `xref.chain-cycle` | The chain of previous sections looped: a `/Prev` or `/XRefStm` names an offset the chain has already named, whether it read a section there or not. Reported at that offset, or, when it lies outside the file, at the section whose trailer named it | | `xref.entry-out-of-range` | An object's entry places it outside the file — past its end, before its start, or past what a `long` holds once the header's offset is added —; reported with no position, the offset in the message, exactly. A section named outside the file is `xref.section-missing`'s, or the rebuild's when `startxref` names it | -| `stream.length-invalid` | A stream's `/Length` is not where its data ends: its `endstream` lies elsewhere and ends the data; or the `/Length` is no length — absent, not a non-negative integer, or naming an object the file lacks or that could not be read — and the `endstream` ends the data; or no `endstream` lies before the next object or the end of the file, no `endobj` of its object ends the data, and the data is the declared length, when it ends before that stop, or runs up to the next object; or the document's searches for `endstream` have read as much of the file as they may, and the data is taken without a search — but where a guard hid the `/Length`, whose data is then the guard's. A `/Length` that names an object a guard cut, or left out of the index, is not judged, though a missing `endstream` is reported all the same. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case. A cross-reference stream whose `/Length` the chain could not read, its data taken up to its first `endstream`, has it read once the chain is, unless a guard cut that data: a `/Length` that is no length, or that no `endstream` confirms, is reported as any stream's, and one an `endstream` past the one the chain stopped at confirms is reported with the rows the chain left unread. Where the section's own row for that `/Length` lies among the rows the data holds, and gives a length an `endstream` confirms, the data is taken again to it before the rows are read, and nothing is reported; nor is anything of a `/Length` a guard cut or left out of the index | +| `stream.length-invalid` | A stream's `/Length` is not where its data ends: its `endstream` lies elsewhere and ends the data; or the `/Length` is no length — absent, not a non-negative integer, or naming an object the file lacks or that could not be read — and the `endstream` ends the data; or no `endstream` lies before the next object or the end of the file, no `endobj` of its object ends the data, and the data is the declared length, when it ends before that stop, or runs up to the next object; or the document's searches for `endstream` have read as much of the file as they may, and the data is taken without a search — but where a guard hid the `/Length`, whose data is then the guard's. A `/Length` that names an object a guard cut, or left out of the index, is not judged, though a missing `endstream` is reported all the same. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case. A cross-reference stream whose `/Length` the chain could not read, its data taken up to its first `endstream`, has it read once the chain is, unless a guard cut that data before every row the section declares — data that holds them all is checked whatever a guard did —: a `/Length` that is no length, or that no `endstream` confirms, is reported as any stream's, and one an `endstream` past the one the chain stopped at confirms is reported with the rows the chain left unread. Where the section's own row for that `/Length` lies among the rows the data holds, and gives a length an `endstream` confirms, the data is taken again to it before the rows are read, and nothing is reported; nor is anything of a `/Length` a guard cut or left out of the index | | `stream.truncated` | A stream has no `endstream` before the next object or the end of the file, and its object's `endobj` ends its data — following the declared length, which is kept, or ending the object before that stop —; or neither, no declared length ends before the end of the file, and its data runs to it. Data taken without a search, the document's searches having read as much of the file as they may, is `stream.length-invalid`'s, to the end of the file too. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case | | `stream.self-reference` | An object stream's dictionary names an object the stream holds — as its `/Type`, `/Length`, `/N`, `/First`, a filter or a parameter —: that object reads as null while the stream is decoded, and the stream is decoded without it | | `object-stream.member-moved` | An object is in the object stream its entry names, at another index than the entry gives, and was read where the stream's header lists it. Reported where the stream's data starts, the object, the stream and both indexes in the message | @@ -128,7 +128,7 @@ Codes are stable: they are part of the public contract, because callers filter o | `limit.object` | An object is longer than `MaxObjectLength`, its stream data aside; the part within it was parsed. An object an object stream holds is bounded alike, and reported once for each definition of its object stream, where that stream's data starts, the object, the stream and the byte of its decoded data where the object starts in the message | | `limit.xref-section-length` | A classic cross-reference table is longer than `MaxXRefSectionLength`; the entries within it were read, and its trailer past them; the objects the rows past them place read as null, and the chain is not followed further | | `limit.xref-section-count` | The chain of cross-reference sections is longer than `MaxXRefSectionCount`; the newest were read, and what only the older ones index reads as null. Reported where the chain goes on, or, when that lies outside the file, at the section whose `/Prev` names it, the offset in the message | -| `limit.trailer` | A trailer, or a cross-reference stream's dictionary — never that stream's data —, is longer than `MaxTrailerLength`; the part within it was parsed, and the chain ends there: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, what only the older sections index reading as null | +| `limit.trailer` | A trailer, or a cross-reference stream's dictionary — never that stream's data —, is longer than `MaxTrailerLength`; the part within it was parsed, and the chain ends there: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, what only the older sections index reading as null. Where the chain names a section and what lies there declares another type before the cut — a CIDFont, whose `/W` gives widths —, the guard cut no section: it is reported all the same, as the dictionary of the object read as a cross-reference stream, and the section is looked for nearby, the chain going on as read whole | | `limit.index-entries` | The index holds `MaxIndexEntries` object numbers — those in use, free or refused — and met another: the numbers it holds were indexed, and an object past them reads as null. Reported once a document, where the first number it had no room for was met: the section that gave it, the object header or object stream a rebuild read, or the chain's newest section for an entry the chain gave that a rebuild could not keep | | `limit.filter-chain` | A stream's `/Filter` array names more filters than `MaxFilterChainLength`, `/Crypt` among them; decoding stopped before the first past them, with what the filters within the bound decoded, and the elements past them were not read. A cross-reference stream or an object stream it ends is read as cut before its data: its rows or its objects are unknown | @@ -190,7 +190,9 @@ it —, or an `endstream` the data spells, an `endobj` after it or not, leaves t reported of the stream, whose data is cut by the guard, and whether its object ends with `endobj` is not judged. An `endstream` is taken for the stream's own, its end the file's, only where what follows its object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places, a cross-reference stream the -chain read, the end of the file, or the `xref` or `startxref` keyword. +chain read, the end of the file, or the `xref` or `startxref` keyword. White space before the keyword, an end-of-line +above all, which only the hidden length says is the data's or not, leaves where the data ends to the guard as it +decodes: a lost tail, a missing checksum or end marker there is not reported, though a whole read reports it. ```text Warning stream.length-invalid at 5501: The stream declared 19954 bytes but ended after 19952. @@ -255,6 +257,8 @@ data within `MaxStreamLength`, or its object within `MaxObjectLength`, or a filt `MaxDecodedStreamLength`. Of data the reader cut, no lost tail, missing checksum or missing end marker is reported, a hexadecimal digit without its pair or an ASCII85 group short of five is dropped rather than completed, and a predictor undoes the part of its last row the data holds rather than report rows the parameters describe as longer than the data. +Nor is any reported of data whose end a guard hides — its `endstream` proven its own past white space that only a +`/Length` a guard cut or left out of the index says is the data's or not —, which decodes as read whole. ```text Warning filter.failed at 59534: A Flate stream ends before its data does; what decoded before the end was kept. diff --git a/docs/website/docs/reference/validation-rules.md b/docs/website/docs/reference/validation-rules.md index 2e5bad05..ed20ae82 100644 --- a/docs/website/docs/reference/validation-rules.md +++ b/docs/website/docs/reference/validation-rules.md @@ -51,8 +51,8 @@ References are to ISO 32000-1:2008. | Rule | Severity | Meaning | Reads | |---|---|---|---| -| `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]); or a cross-reference stream whose `/Length` is such a reference, or leads through references to an object no section read before it places, and whose data no `endstream` ends before the next object the sections read so far place, or the end of the file — or was not searched for one, the document's searches for `endstream` having read as much of the file as they may ([#220]). Rows that spell `endstream` end such a stream's data there, and the section lacks the rows past them, unless the section's own row for that `/Length` — the row the reading of its rows keeps, one of a reserved type ignored —, among the rows before them, places an object that gives a longer length an `endstream` confirms: the data is then taken to it. A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. Nor is a cross-reference stream whose value the rows are read with leads to an object a limit cut: the section counts as cut there, the chain ends at it, nothing is rebuilt, and `xref.checked-in-part` says so ([#263]). Nor is one whose `/Length`, which the chain could not read, leads to an object `MaxObjectLength` cuts, where only that length could say where the data ends: with no `endstream` before the next object or the end of the file, the section counts as cut, as above; when its rows spell one, the integer the cut left of that object — whether the section's own row places it, or a newer section does — is where an `endstream` is looked for, the data taken to it when one confirms it, and otherwise the rows past the spelled one the limit's, the section counting as cut there, as above ([#224]). | Each section of the chain | -| `xref.section-not-found` | Error | A section a `/Prev` or an `/XRefStm` names is neither there nor within 512 bytes of it, or what names it is not an offset — tiff2pdf writes `/Prev 576066 0 R`. The objects only it indexes are missing from the index. The section `startxref` names is `file.startxref-wrong`'s. A `/Prev` or an `/XRefStm` written as a reference that leads to an object `MaxObjectLength` cuts names no section missing: the chain ends there for the limit, what it names unread, which `xref.checked-in-part` says ([#263]). | Each section of the chain | +| `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]); or a cross-reference stream whose `/Length` is such a reference, or leads through references to an object no section read before it places, and whose data no `endstream` ends before the next object the sections read so far place, or the end of the file — or was not searched for one, the document's searches for `endstream` having read as much of the file as they may ([#220]). Rows that spell `endstream` end such a stream's data there, and the section lacks the rows past them, unless the section's own row for that `/Length` — the row the reading of its rows keeps, one of a reserved type ignored —, among the rows before them, places an object that gives a longer length an `endstream` confirms: the data is then taken to it. A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. Nor is a cross-reference stream whose value the rows are read with leads to an object a limit cut: the section counts as cut there, the chain ends at it, nothing is rebuilt, and `xref.checked-in-part` says so ([#263]) — but for what makes it unreadable whatever the value holds, as read whole: a value past it no section read so far places, a `/W` that gives no three widths or none past 0, or rows numbered wrongly as far as the values read say; and a stream that declares another type is no section a limit cut a value of. Nor is one whose `/Length`, which the chain could not read, leads to an object `MaxObjectLength` cuts, where only that length could say where the data ends: with no `endstream` before the next object or the end of the file, the section counts as cut, as above; when its rows spell one, the integer the cut left of that object — whether the section's own row places it, or a newer section does — is where an `endstream` is looked for, the data taken to it when one confirms it, and otherwise the rows past the spelled one the limit's, the section counting as cut there, as above ([#224]). | Each section of the chain | +| `xref.section-not-found` | Error | A section a `/Prev` or an `/XRefStm` names is neither there nor within 512 bytes of it, or what names it is not an offset — tiff2pdf writes `/Prev 576066 0 R`. The objects only it indexes are missing from the index. The section `startxref` names is `file.startxref-wrong`'s. A `/Prev` or an `/XRefStm` the reader follows, written as a reference that leads to an object `MaxObjectLength` cuts, names no section missing: the chain ends there for the limit, what it names unread, which `xref.checked-in-part` says ([#263]). | Each section of the chain | | `xref.section-shifted` | Warning | A section a `/Prev` or an `/XRefStm` names starts a few bytes from where it is named, and the reader found it there. | Each section of the chain | | `xref.chain-loop` | Error | The chain names a section it has already read, or an offset it has already named where it read none, in the file or outside it: the reader stops there, and the section the chain should have gone on to is unknown, what only it indexed found, if at all, by rebuilding the index — as for `xref.section-not-found`. | The chain | | `xref.entry-broken` | Error | An in-use entry places its object neither at its offset nor within 512 bytes of it, in an object stream whose header does not list it, or in one where it is written as a stream or holds one, which an object stream cannot hold (7.5.7), and which reads as null — the index the reader rebuilt included, where it places such an object and the file's own index places nothing, or the part of it the reader read where one of its limits stopped the chain short ([#161]) — as a load decides it: the header read at the offset after white space and comments, however long, a whole token no header begins with ruling one out at once, and the one found near it confirmed by what a load parses there ([#175], [#256]). An offset outside the file — past its end, before its start, or past what a `long` holds once the header's offset is added — is looked for nearby as a load looks: an object found there is `xref.entry-shifted`'s. One finding per entry, located at the object and the offset the entry gives, or at the object alone when that offset lies outside the file. | Each object's header, as a load reads it, a few dozen bytes in a sound file; each object stream's header, and each member whose bytes hold `stream`, parsed as a load parses it | @@ -63,7 +63,7 @@ References are to ISO 32000-1:2008. | `xref.object-stream-value-wrong` | Warning | An object stream gives its `/N`, `/First` or `/Length` as a real number with no fractional part, where the specification asks for an integer (Tables 5 and 16). The reader reads it as the integer it equals, as MuPDF, PDFBox and pdf.js do; qpdf and poppler lose the stream's objects ([#215]). One finding per stream and key. A value with a fractional part is another fault: a `/N` or a `/First` leaves the stream unreadable, `xref.object-stream-broken`'s, and a `/Length` is recovered through the stream's `endstream`, which the reader reports (`stream.length-invalid`). | As `xref.object-stream-circular` | | `xref.offset-imprecise` | Warning | Offsets name the white space before what they designate rather than its first byte (7.5.4, 7.5.5): Microsoft Print to PDF names the line feed before each object, and some tools the one before `xref`. Every reader skips the white space. One finding for the file, with how many offsets and the first of them. | The sections of the chain, and each object's header | | `xref.object-past-size` | Warning | In-use objects are numbered above the trailer's `/Size`, which Table 15 makes a conforming reader ignore ("shall be ignored and defined to be missing"). The entries, the objects' headers and the references agree against `/Size`, and the reader reads those objects, as qpdf does while warning. One finding for the file, with how many objects and the first of them; `file.size-wrong` then says nothing more of that `/Size`. | The index and the trailer's `/Size` | -| `xref.checked-in-part` | Information | Part of the index was not checked: one of the reader's limits stopped it reading a section, a trailer or an object stream whole, the chain past `MaxXRefSectionCount` sections, or the index past the `MaxIndexEntries` object numbers it holds, said once, where it filled — raising the limit lets the rest be checked —; an entry whose header, where it places its object or near it, runs past `MaxObjectLength`; the entries the probe left undecided once reading their headers had consumed twice the file's length — the bytes crossed up to each decision, past the first 64 of each, and every byte read past a run an entry skips, which a valid file's headers, distinct bytes, never reach —, said once ([#175]); the sections older than one whose `/Prev` leads to an object `MaxObjectLength` cuts — or the cross-reference stream of its revision, and any section older than it, when its `/XRefStm` does —, which end the chain there, said at that section with the reference that names them ([#263]); or the document is encrypted and its object streams are readable only once decrypted (M16). A chain cut where it goes on outside the file is located at the section whose `/Prev` names where. | What the reader and the probe of the index recorded | +| `xref.checked-in-part` | Information | Part of the index was not checked: one of the reader's limits stopped it reading a section, a trailer or an object stream whole, the chain past `MaxXRefSectionCount` sections, or the index past the `MaxIndexEntries` object numbers it holds, said once, where it filled — raising the limit lets the rest be checked —; an entry whose header, where it places its object or near it, runs past `MaxObjectLength`; the entries the probe left undecided once reading their headers had consumed twice the file's length — the bytes crossed up to each decision, past the first 64 of each, and every byte read past a run an entry skips, which a valid file's headers, distinct bytes, never reach —, said once ([#175]); the sections older than one whose `/Prev`, which the reader follows, leads to an object `MaxObjectLength` cuts — or the cross-reference stream of its revision, and any section older than it, when its `/XRefStm` does —, which end the chain there, said at that section with the reference that names them ([#263]); or the document is encrypted and its object streams are readable only once decrypted (M16). A chain cut where it goes on outside the file is located at the section whose `/Prev` names where. | What the reader and the probe of the index recorded | The entries the `xref.entry-*`, `xref.generation-mismatch`, `xref.object-stream-*`, `xref.offset-imprecise` and `xref.object-past-size` rules check are those of the index the file's chain gave, as the reader first read @@ -87,7 +87,7 @@ it is checked: the exception above judges the index the reader rebuilt, and the | Rule | Severity | Meaning | Reads | |---|---|---|---| | `object.reference-missing` | Warning | An object reachable from the trailer refers to an object the file does not define — none in its index, a free entry, or object 0, which heads the free list and is never in use (7.5.4) —, which is null (7.3.10): the reader reads it so, rebuilding nothing, and what the reference was to give is lost. One finding per object that holds such references, naming the first. An object one of the reader's limits left out of the index is unknown rather than missing, and is `xref.checked-in-part`'s. An object the index holds and the reader cannot produce is `xref.entry-broken`'s or `xref.object-stream-broken`'s — but one a rebuild serves from another object stream than the chain's index places it in, or from another definition of that stream, which does not list it or holds it written as a stream, and so loses, which no finding names yet ([#276]) —, a kid of the page tree `page-tree.kid-invalid`'s — one in a node's `/Kids`, an array of kids of its own or a root written in the catalog the walk read whole, not what an array written among the kids holds, which the walk does not read —, and a trailer's `/Root` that names nothing `file.root-invalid`'s. | Every object reachable from the trailer, once each | -| `object.endobj-missing` | Warning | An object reachable from the trailer does not end with `endobj` (7.3.10): another token, or the end of the file, follows its value. The reader reads the value as far as it goes, as qpdf does while reporting it. An empty object, `2 0 obj endobj`, has its `endobj`. An object whose array or dictionary was never closed lacks it where the next object's header — `N G obj` in digits, white space alone between them, an object number from 1 to 2,147,483,647 and a generation up to 65,535 —, met where an element, a key or a value is expected, ends every container left open ([#230]); an array or a dictionary that holds such a header and is closed after it ends there all the same, its `endobj` judged missing, which none of the corpus's 344,361 objects does. What follows a stream's `endstream` is read through white space and comments however long, and the file asked for it when it lies past the 8 KB window the reader first reads through — after a stream whose data runs past that window ([#55]), or an `endstream` the window's edge comes too soon after ([#174]). A stream with no `endstream` has its `endobj` when the object's `endobj` follows the length it declared — after white space and comments, a comment only once white space comes before it —, or ends its data with nothing but white space and comments after it before the next object or the end of the file; one whose data runs on to the next object, or to the end of the file, lacks it. Where the object ends is unknown, and is not judged, after a stream whose declared length the reader kept with neither an `endstream` nor an `endobj` after it, one whose data, run on to the next object or the end of the file, holds an `endobj` that may be its object's own, and one whose end lies past what the reader reads first once the document's searches for `endstream` have read as much of the file as they may. So is a stream whose `/Length` one of the reader's limits cut or left out of the index, when an object header its data spells, or an `endstream` it spells, ends its data: only that length could tell either from the stream's own end. An `endstream` is the stream's own, and an `endobj` its object lacks judged missing, only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places, the end of the file, or the `xref` or `startxref` keyword ([#224]). | What follows each reachable object's value, as the reader read it | +| `object.endobj-missing` | Warning | An object reachable from the trailer does not end with `endobj` (7.3.10): another token, or the end of the file, follows its value. The reader reads the value as far as it goes, as qpdf does while reporting it. An empty object, `2 0 obj endobj`, has its `endobj`. An object whose array or dictionary was never closed lacks it where the next object's header — `N G obj` in digits, white space alone between them, an object number from 1 to 2,147,483,647 and a generation up to 65,535 —, met where an element, a key or a value is expected, ends every container left open ([#230]); an array or a dictionary that holds such a header and is closed after it ends there all the same, its `endobj` judged missing, which none of the corpus's 344,361 objects does. What follows a stream's `endstream` is read through white space and comments however long, and the file asked for it when it lies past the 8 KB window the reader first reads through — after a stream whose data runs past that window ([#55]), or an `endstream` the window's edge comes too soon after ([#174]). A stream with no `endstream` has its `endobj` when the object's `endobj` follows the length it declared — after white space and comments, a comment only once white space comes before it —, or ends its data with nothing but white space and comments after it before the next object or the end of the file; one whose data runs on to the next object, or to the end of the file, lacks it. Where the object ends is unknown, and is not judged, after a stream whose declared length the reader kept with neither an `endstream` nor an `endobj` after it, one whose data, run on to the next object or the end of the file, holds an `endobj` that may be its object's own, and one whose end lies past what the reader reads first once the document's searches for `endstream` have read as much of the file as they may. So is a stream whose `/Length` one of the reader's limits cut or left out of the index, when an object header its data spells, or an `endstream` it spells, ends its data: only that length could tell either from the stream's own end. An `endstream` is the stream's own, and an `endobj` its object lacks judged missing, only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places, a cross-reference stream the chain read, the end of the file, or the `xref` or `startxref` keyword ([#224]). | What follows each reachable object's value, as the reader read it | | `object.name-null-character` | Warning | An object reachable from the trailer holds a name with a null character, written `#00`, which a name cannot contain (7.3.5). The reader keeps the name; qpdf refuses it and reads dictionaries without the keys so named. One finding per object, naming the first. | Every name of every reachable object | | `object.key-missing` | Warning | An object reachable from the trailer lacks a key the [Arlington PDF Model](#object-shapes-from-the-arlington-pdf-model) requires of its type — absent, or `null`, which in a dictionary is the same (7.3.7): a key the model requires once the version the file declares has it, or one it requires in some versions only, such as a form XObject's `/Name` in PDF 1.0. An inheritable key is looked for up the `/Parent` chain, and a field's `/DA` in the interactive form's too; a key only an extension of ISO 32000 defines is never required. An array shorter than the elements it requires lacks each required element it does not hold, one finding per element. [One finding per type and key](#object-shapes-from-the-arlington-pdf-model), at the first object, with how many lack it. | Every object reachable from the trailer, typed along the model's links, once each | | `object.value-type-wrong` | Warning | A value is of a type the Arlington PDF Model does not allow for its key, judged by the value's class: a real, even `400.0`, where an integer belongs; a string where a name does; a stream where a dictionary does, or the reverse, which is reported on the object that holds it and not walked into. A `null` written in an array is judged like any other value; one reached through a reference is not judged. One finding per type and key. | As `object.key-missing` | diff --git a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs index 60b3b088..e0d9253e 100644 --- a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs +++ b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs @@ -39,7 +39,9 @@ public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diag /// Decodes a stream's data under its document's guards, and says whether one of them ended decoding — the most filters /// the reader lets data pass through, or a value the filters read that a guard cut or left out of the index: what comes /// back is then the data still encoded by the filters past it, no part of what the stream decodes to —, and whether one - /// cut the data, as a reader of its rows or its objects must know. + /// cut the data, as a reader of its rows or its objects must know. Data a proven endstream ended past white space + /// that only a hidden length says is the data's or not () decodes as read whole, + /// and a filter that runs out at its end says nothing of the file. /// /// The stream to decode. /// Receives what decoding met; when none are supplied, the stream's document does. diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index b7e99ca9..c02c4a33 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -145,11 +145,11 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// /// /// Not a guard a valid file reaches (ADR 34), argued here once for every read it counts. A valid file is read past - /// the parser's window in four ways only. Three read the data: a cross-reference stream whose /Length the - /// chain cannot read (#220) is searched once to its endstream; a stream whose /Length a guard cut or - /// left out of the index (#224) is searched once to its endstream, or to a header its data spells, which - /// leaves the data up to it to the guard; and white space after a declared length that runs past a first read is - /// read on once for each place (#174). The data of different streams lies apart, and so do the runs of white space + /// the parser's window in four ways only. Three read the data or what follows it: a cross-reference stream whose + /// /Length the chain cannot read (#220) is searched once to its endstream; a stream whose + /// /Length a guard cut or left out of the index (#224) is searched once to its endstream, or to a + /// header its data spells, which leaves the data up to it to the guard; and white space after a declared length + /// that runs past a first read is read on once for each place (#174). The data of different streams lies apart, and so do the runs of white space /// after them: together those reads cover the file once, and one read past each — about twice the file at most. The /// fourth adds a little to each stream of the second way: the proof that an endstream found is the stream's /// own (#224) reads past it to its object's end and what follows, three first reads of diff --git a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs index b4a1a9de..a92ad7a9 100644 --- a/src/AdCodicem.Pdf/IO/PdfObjectParser.cs +++ b/src/AdCodicem.Pdf/IO/PdfObjectParser.cs @@ -835,8 +835,9 @@ private PdfStream ReadStream(PdfDictionary dictionary) // Where a guard cut the /Length or left it out of the index, an endstream the data spells, as a valid stream's may — // in a string, followed by endobj or not —, ends the search as the stream's own would: the length that tells them - // apart is the guard's. One what follows proves its own — the next object the index places, the end of the file, - // a cross-reference section's keyword, after the object's endobj or in its place — ends the data, the file's. + // apart is the guard's. One what follows proves its own — the next object the index places, a cross-reference + // stream the chain read, the end of the file, a cross-reference section's keyword, after the object's endobj or + // in its place — ends the data, the file's. // Any other's data is the guard's, cut by it, the object's endobj unjudged, and nothing is said of the file, as // at a header the data spells (#224): one past which the searches could read nothing more, too. if (hidden && _streamData is { } file && diff --git a/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs b/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs index 6ca20f92..8d9911b3 100644 --- a/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs +++ b/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs @@ -76,7 +76,8 @@ public XRefSectionRecord(string namedBy, long namedOffset, long namedFrom) /// /// Gets a value indicating whether what cut the section is a value it is read with — one its rows are read with, or its /// /Length — that leads to an object one of the reader's guards cut: the section was found whole, and is the - /// guard's to keep from being read (#263). A guard that cut the section's own dictionary leaves it unknown. + /// guard's to keep from being read (#263). A guard that cut the section's own dictionary leaves it unknown, but for + /// what declared itself a section before the cut (, or a table's xref keyword). /// public bool CutByValue { get; set; } From 2122485830aa4c8a716ac2ede210ea92c0112670 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 20:05:17 +0000 Subject: [PATCH 119/146] fix: leave the guard what a cut subsection element or a stream's own type hides A stream with a /W where the chain names a section is read as the section whole, whatever /Type it gives: one that misspells XRef reads clean. Taking a stream of another type for no section once a guard cut a value of it charged the file with a section it could not read, or found an older revision's stream nearby and served objects an update replaced. Such a section is the guard's again, as the value hides whether its rows are. A subsection one of whose two elements a guard cut was not judged, though the other one is no integer, a count below 0 or past the object numbers, or a first number outside them, wrong whatever the cut one holds: it is the file's fault, as read whole. Tests hold what the values read beside a cut leave to the guard. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 6 +- docs/website/docs/reference/diagnostics.md | 4 +- docs/website/docs/reference/reader-limits.md | 13 ++- .../docs/reference/validation-rules.md | 2 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 85 +++++++++++----- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 97 ++++++++++++++++--- 6 files changed, 160 insertions(+), 47 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index d8741a00..97a2175b 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -283,7 +283,7 @@ Known gaps: | A classic table of any length | Its window grows ×4 from 64 KB up to `MaxXRefSectionLength`, and the rows read are kept. A table the guard cuts, or whose rows run past a full index, keeps its trailer, found past the rows left unread by a search a megabyte at a time (`ReadTrailerPastRows`): comments are skipped, and anything else no row holds is the section's fault, reported, the search going on to the `trailer` keyword, so that its `/Encrypt` still refuses the document. The search starts at the start of the line the cut fell in, outside any comment and past the table's own keyword; it reads each megabyte once, every byte of it looked at before the next is read, however many comments or stray bytes it holds, and is not made for a place a section may have been moved to, a guess. The chain ends there, since an older section's rows would stand for the objects the unread rows define. What those rows place reads as null until the guard is raised | `ReaderLimitsTests.A_table_whose_keywords_the_bound_cuts_reports_the_bound`, `A_table_its_length_bound_cuts_keeps_its_trailer_and_the_encryption_it_names`, `A_table_its_length_bound_cuts_ends_the_chain_though_its_trailer_names_an_older_section`, `What_ends_the_rows_a_length_bound_cut_is_judged_as_it_is_where_none_did`, `A_table_a_length_bound_cuts_that_runs_to_the_end_of_the_file_has_no_trailer`, `A_table_its_length_bound_cuts_keeps_the_encryption_its_trailer_names_past_what_no_row_holds`, `A_table_cut_before_its_keyword_ends_keeps_its_trailer`, `Rows_past_a_full_index_are_not_read_and_the_trailer_past_them_is`, `A_cut_in_a_comment_among_the_rows_searches_for_the_trailer_from_the_comment_s_start`, `The_trailer_past_rows_left_unread_is_found_wherever_the_search_s_window_ends`, `A_trailer_past_rows_left_unread_that_its_own_bound_cuts_keeps_what_precedes_the_cut`; `DocumentReaderTests.The_search_for_the_trailer_past_rows_left_unread_reads_the_file_once`; `CrossReferenceChainTests.Reads_an_object_a_length_bound_cut_out_of_its_table_as_null_until_the_guard_is_lifted`, `A_section_found_near_where_prev_names_it_is_not_searched_past_the_rows_a_guard_left_unread`; `WindowEdgeTests.A_classic_table_reads_whole_wherever_its_window_edge_falls_near_its_trailer` | No: seeds stay under 120 KB | | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). A hybrid file's stream's own `/Prev`, which the chain does not follow, going on through the table's, is not read at all, nor are the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place: neither names a section missing nor ends the chain for a guard, and the rule on trailer values judges them as written, a reference they hold not followed ([#263]). An object the guard cut after a whole reference it holds hides nothing: the chain reads through it, as a load does, and loads it once the chain is read. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s`, `A_table_an_xrefstm_names_in_a_stream_s_place_has_its_own_links_left_unread`, `A_prev_written_as_a_reference_to_a_real_is_judged_through_it_only_where_the_reader_follows_it` | Yes | -| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); so is one that declared itself a section before a guard cut it — a table's `xref` keyword, `/Type /XRef`. A place that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is a guess dropped, the next place tried; where the chain names a section, a guard that cut what declares another type — its dictionary or a value of it — cut no section, and relocation is tried as read whole. What makes a section unreadable whatever a value a guard cut holds is the file's fault, as read whole, judged before the cut is taken for the guard's (`PdfFileReader.FaultBesideCut`): a value past the cut that no section read so far places, which the sections a whole read reads before make the file's to lack; a `/W` that gives no three widths, or none past 0; rows numbered wrongly — an `/Index` that holds no pairs, whatever its elements, a subsection out of range, a `/Size` with no `/Index` that gives no count —, as far as the values read, written direct or through references, say: a value a guard cut says nothing, nor does what depends on it, the `/Size` an `/Index` so cut may stand beside | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_cross_reference_stream_whose_deferred_length_a_guard_cut_is_found_where_it_was_moved`, `A_cross_reference_stream_a_guard_cut_the_dictionary_of_after_its_type_is_the_section_where_it_was_moved`, `A_classic_table_a_guard_cut_the_rows_of_is_the_section_where_it_was_moved`, `A_stream_with_no_numbering_whose_width_a_guard_cut_is_no_section_the_guard_cut`, `A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild`, `What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_guard_cut_holds_is_the_file_s`, `A_place_a_section_may_have_moved_to_that_a_value_past_a_guard_s_cut_makes_unreadable_is_dropped`, `A_stream_of_another_type_whose_value_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut` | No | +| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); so is one that declared itself a section before a guard cut it — a table's `xref` keyword, `/Type /XRef`. A place that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is a guess dropped, the next place tried; where the chain names a section, a guard that cut the dictionary of what declares another type cut no section — a CIDFont's, which read whole no stream data follows —, and relocation is tried, while one that cut a value of a stream there leaves it the section the guard cut, whatever type it gives, as read whole it is the section. What makes a section unreadable whatever a value a guard cut holds is the file's fault, as read whole, judged before the cut is taken for the guard's (`PdfFileReader.FaultBesideCut`): a value past the cut that no section read so far places, which the sections a whole read reads before make the file's to lack; a `/W` that gives no three widths, or none past 0; rows numbered wrongly — an `/Index` that holds no pairs, whatever its elements, a subsection out of range, or wrong by the element read whatever the cut one holds, a `/Size` with no `/Index` that gives no count —, as far as the values read, written direct or through references, say: a value a guard cut says nothing, nor does what depends on it, the `/Size` an `/Index` so cut may stand beside | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_cross_reference_stream_whose_deferred_length_a_guard_cut_is_found_where_it_was_moved`, `A_cross_reference_stream_a_guard_cut_the_dictionary_of_after_its_type_is_the_section_where_it_was_moved`, `A_classic_table_a_guard_cut_the_rows_of_is_the_section_where_it_was_moved`, `A_stream_with_no_numbering_whose_width_a_guard_cut_is_no_section_the_guard_cut`, `A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild`, `What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_guard_cut_holds_is_the_file_s`, `A_place_a_section_may_have_moved_to_that_a_value_past_a_guard_s_cut_makes_unreadable_is_dropped`, `A_stream_whose_value_a_guard_cut_where_the_chain_names_a_section_is_the_section_the_guard_cut_whatever_its_type`, `A_cross_reference_stream_that_misspells_its_type_and_whose_value_a_guard_cut_is_the_section_the_guard_cut`, `A_section_a_value_s_cut_keeps_from_being_read_is_the_guard_s_where_nothing_read_says_it_is_wrong` | No | | A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, may stay within the bound — the group counted against what the filters before the last share as well —: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_decoding_guard_deferred_data_reached_is_reported_once_whatever_room_the_diagnostics_have`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `Data_whose_end_a_guard_hides_leaves_the_filters_after_a_dropped_group_the_room_its_direct_twin_s_have`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file`, `A_cross_reference_stream_the_chain_read_proves_the_endstream_of_the_older_one_before_it` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | @@ -385,6 +385,10 @@ Known gaps: the guard's: a Flate stream that lost its tail or its checksum, or ASCII85, ASCIIHex, RunLength or LZW data without its end marker, is not reported there, though a whole read reports it; only the hidden length says whether that white space is the data's ([#224]). +- Accepted, with no issue: where the chain names a section, an object that declares another type and gives a `/W` + before `MaxTrailerLength` cuts its dictionary is taken for no section, as a CIDFont's dictionary is read whole, no + stream data after it, and the section is looked for nearby; a stream of that type, whose stream data the cut hides, + would be read as the section whole ([#263]). ### Decoding diff --git a/docs/website/docs/reference/diagnostics.md b/docs/website/docs/reference/diagnostics.md index f8fc0722..3f1b0c4d 100644 --- a/docs/website/docs/reference/diagnostics.md +++ b/docs/website/docs/reference/diagnostics.md @@ -102,7 +102,7 @@ Codes are stable: they are part of the public contract, because callers filter o | `xref.rebuilt` | The index was rebuilt by scanning the whole file | | `xref.offset-adjusted` | An object, or a cross-reference section the chain names, was not where the file said, and was found nearby | | `xref.section-missing` | A cross-reference section `/Prev` or `/XRefStm` names holds nothing that reads as a section, nor does any place nearby, lies outside the file, or is named by a value that is not an offset; what only it indexed is found by rebuilding the index when it is asked for. Reported at the offset the section was named at, or, when that offset lies outside the file or the value is not an offset, at the section whose trailer named it. A `/Prev` or `/XRefStm` the reader follows — not a hybrid file's cross-reference stream's own `/Prev`, nor the links of a table an `/XRefStm` names, which it leaves unread —, written as a reference that leads to an object one of the reader's limits cut, names what the guard hides: no section is reported missing, nothing is rebuilt, the chain ends there, and the guard is reported once the chain is read | -| `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, data or — with no `/Index` — `/Size` cannot give rows, or whose `/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, is a reference no section read before it places where it can be read without loading it, or whose `/Length` is such a reference, or leads to one through others, and no `endstream` follows its data before the next object the sections read so far place, or before the end of the file, or none was looked for once the document's searches for `endstream` read as much of the file as they may — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message. A value that names an object one of the reader's limits cut, directly or through a chain of references — a `/Length` whose data no `endstream` ends before that stop among them —, is the guard's, not the file's: the section is not reported, nor looked for nearby, the chain ends there, nothing is rebuilt, what only that section and the older ones index reads as null, and the guard is reported once the chain is read — but for what makes the section unreadable whatever the value holds, which is the file's and reported here, as read whole: a value past it no section read so far places, a `/W` that gives no three widths or none past 0, or rows numbered wrongly as far as the values read say; and an object that declares another type, a font say, is no section a guard cut a value of, and the section is looked for nearby | +| `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, data or — with no `/Index` — `/Size` cannot give rows, or whose `/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, is a reference no section read before it places where it can be read without loading it, or whose `/Length` is such a reference, or leads to one through others, and no `endstream` follows its data before the next object the sections read so far place, or before the end of the file, or none was looked for once the document's searches for `endstream` read as much of the file as they may — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message. A value that names an object one of the reader's limits cut, directly or through a chain of references — a `/Length` whose data no `endstream` ends before that stop among them —, is the guard's, not the file's: the section is not reported, nor looked for nearby, the chain ends there, nothing is rebuilt, what only that section and the older ones index reads as null, and the guard is reported once the chain is read — but for what makes the section unreadable whatever the value holds, which is the file's and reported here, as read whole: a value past it no section read so far places, a `/W` that gives no three widths or none past 0, or rows numbered wrongly as far as the values read say — a subsection wrong by the element read whatever the cut one holds among them | | `xref.chain-cycle` | The chain of previous sections looped: a `/Prev` or `/XRefStm` names an offset the chain has already named, whether it read a section there or not. Reported at that offset, or, when it lies outside the file, at the section whose trailer named it | | `xref.entry-out-of-range` | An object's entry places it outside the file — past its end, before its start, or past what a `long` holds once the header's offset is added —; reported with no position, the offset in the message, exactly. A section named outside the file is `xref.section-missing`'s, or the rebuild's when `startxref` names it | | `stream.length-invalid` | A stream's `/Length` is not where its data ends: its `endstream` lies elsewhere and ends the data; or the `/Length` is no length — absent, not a non-negative integer, or naming an object the file lacks or that could not be read — and the `endstream` ends the data; or no `endstream` lies before the next object or the end of the file, no `endobj` of its object ends the data, and the data is the declared length, when it ends before that stop, or runs up to the next object; or the document's searches for `endstream` have read as much of the file as they may, and the data is taken without a search — but where a guard hid the `/Length`, whose data is then the guard's. A `/Length` that names an object a guard cut, or left out of the index, is not judged, though a missing `endstream` is reported all the same. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case. A cross-reference stream whose `/Length` the chain could not read, its data taken up to its first `endstream`, has it read once the chain is, unless a guard cut that data before every row the section declares — data that holds them all is checked whatever a guard did —: a `/Length` that is no length, or that no `endstream` confirms, is reported as any stream's, and one an `endstream` past the one the chain stopped at confirms is reported with the rows the chain left unread. Where the section's own row for that `/Length` lies among the rows the data holds, and gives a length an `endstream` confirms, the data is taken again to it before the rows are read, and nothing is reported; nor is anything of a `/Length` a guard cut or left out of the index | @@ -128,7 +128,7 @@ Codes are stable: they are part of the public contract, because callers filter o | `limit.object` | An object is longer than `MaxObjectLength`, its stream data aside; the part within it was parsed. An object an object stream holds is bounded alike, and reported once for each definition of its object stream, where that stream's data starts, the object, the stream and the byte of its decoded data where the object starts in the message | | `limit.xref-section-length` | A classic cross-reference table is longer than `MaxXRefSectionLength`; the entries within it were read, and its trailer past them; the objects the rows past them place read as null, and the chain is not followed further | | `limit.xref-section-count` | The chain of cross-reference sections is longer than `MaxXRefSectionCount`; the newest were read, and what only the older ones index reads as null. Reported where the chain goes on, or, when that lies outside the file, at the section whose `/Prev` names it, the offset in the message | -| `limit.trailer` | A trailer, or a cross-reference stream's dictionary — never that stream's data —, is longer than `MaxTrailerLength`; the part within it was parsed, and the chain ends there: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, what only the older sections index reading as null. Where the chain names a section and what lies there declares another type before the cut — a CIDFont, whose `/W` gives widths —, the guard cut no section: it is reported all the same, as the dictionary of the object read as a cross-reference stream, and the section is looked for nearby, the chain going on as read whole | +| `limit.trailer` | A trailer, or a cross-reference stream's dictionary — never that stream's data —, is longer than `MaxTrailerLength`; the part within it was parsed, and the chain ends there: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, what only the older sections index reading as null. Where the chain names a section and what lies there declares another type before the cut — a CIDFont, whose `/W` gives widths —, the guard cut no section: it is reported all the same, as the dictionary of the object read as a cross-reference stream, and the section is looked for nearby, the chain going on, as for a CIDFont's dictionary read whole, which no stream data follows | | `limit.index-entries` | The index holds `MaxIndexEntries` object numbers — those in use, free or refused — and met another: the numbers it holds were indexed, and an object past them reads as null. Reported once a document, where the first number it had no room for was met: the section that gave it, the object header or object stream a rebuild read, or the chain's newest section for an entry the chain gave that a rebuild could not keep | | `limit.filter-chain` | A stream's `/Filter` array names more filters than `MaxFilterChainLength`, `/Crypt` among them; decoding stopped before the first past them, with what the filters within the bound decoded, and the elements past them were not read. A cross-reference stream or an object stream it ends is read as cut before its data: its rows or its objects are unknown | diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index 98315d51..7859fafe 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -140,11 +140,14 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. offset it is named at, which such a value keeps from being read, or which declared itself one — a table's `xref` keyword, `/Type /XRef` — before a guard cut it; a place there that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is not taken for the section, and the next is - tried. A guard that cut, where the chain names a section, what declares another type — its dictionary or a value of it - — cut no section; and what makes a section unreadable whatever a value a guard cut holds is the file's fault, as read - whole: a value past the cut that no section read so far places, a `/W` that gives no three widths or none past 0, or - rows numbered wrongly — an `/Index` that holds no pairs, or a subsection, or a `/Size` with no `/Index`, out of - range —, as far as the values read, written direct or through references, say. The rows a cross-reference stream's data lacks, + tried. A guard that cut, where the chain names a section, the dictionary of what declares another type cut no section — + a CIDFont's, which read whole no stream data follows —, while one that cut a value of a stream there leaves it the + section the guard cut, whatever type it gives, as read whole it is the section. What makes a section unreadable whatever + a value a guard cut holds is the file's fault, as read whole: a value past the cut that no section read so far places, a + `/W` that gives no three widths or none past 0, or rows numbered wrongly — an `/Index` that holds no pairs, a + subsection that numbers no object or numbers past the object numbers, by both its elements or by the one read whatever + the cut one holds, a `/Size` with no `/Index` that gives no count —, as far as the values read, written direct or + through references, say. The rows a cross-reference stream's data lacks, where its `/Length` so leads and its data was taken to the first `endstream`, are the guard's too, unless what follows that `endstream` proves it the stream's own, as above: they are then the file's to lack, and the index is rebuilt. An object the chain read through a reference the cut left whole is loaded once the chain is read only for a value it diff --git a/docs/website/docs/reference/validation-rules.md b/docs/website/docs/reference/validation-rules.md index ed20ae82..d99e4a24 100644 --- a/docs/website/docs/reference/validation-rules.md +++ b/docs/website/docs/reference/validation-rules.md @@ -51,7 +51,7 @@ References are to ISO 32000-1:2008. | Rule | Severity | Meaning | Reads | |---|---|---|---| -| `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]); or a cross-reference stream whose `/Length` is such a reference, or leads through references to an object no section read before it places, and whose data no `endstream` ends before the next object the sections read so far place, or the end of the file — or was not searched for one, the document's searches for `endstream` having read as much of the file as they may ([#220]). Rows that spell `endstream` end such a stream's data there, and the section lacks the rows past them, unless the section's own row for that `/Length` — the row the reading of its rows keeps, one of a reserved type ignored —, among the rows before them, places an object that gives a longer length an `endstream` confirms: the data is then taken to it. A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. Nor is a cross-reference stream whose value the rows are read with leads to an object a limit cut: the section counts as cut there, the chain ends at it, nothing is rebuilt, and `xref.checked-in-part` says so ([#263]) — but for what makes it unreadable whatever the value holds, as read whole: a value past it no section read so far places, a `/W` that gives no three widths or none past 0, or rows numbered wrongly as far as the values read say; and a stream that declares another type is no section a limit cut a value of. Nor is one whose `/Length`, which the chain could not read, leads to an object `MaxObjectLength` cuts, where only that length could say where the data ends: with no `endstream` before the next object or the end of the file, the section counts as cut, as above; when its rows spell one, the integer the cut left of that object — whether the section's own row places it, or a newer section does — is where an `endstream` is looked for, the data taken to it when one confirms it, and otherwise the rows past the spelled one the limit's, the section counting as cut there, as above ([#224]). | Each section of the chain | +| `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]); or a cross-reference stream whose `/Length` is such a reference, or leads through references to an object no section read before it places, and whose data no `endstream` ends before the next object the sections read so far place, or the end of the file — or was not searched for one, the document's searches for `endstream` having read as much of the file as they may ([#220]). Rows that spell `endstream` end such a stream's data there, and the section lacks the rows past them, unless the section's own row for that `/Length` — the row the reading of its rows keeps, one of a reserved type ignored —, among the rows before them, places an object that gives a longer length an `endstream` confirms: the data is then taken to it. A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. Nor is a cross-reference stream whose value the rows are read with leads to an object a limit cut: the section counts as cut there, the chain ends at it, nothing is rebuilt, and `xref.checked-in-part` says so ([#263]) — but for what makes it unreadable whatever the value holds, as read whole: a value past it no section read so far places, a `/W` that gives no three widths or none past 0, or rows numbered wrongly as far as the values read say — a subsection wrong by the element read whatever the cut one holds among them —, whatever type the stream gives. Nor is one whose `/Length`, which the chain could not read, leads to an object `MaxObjectLength` cuts, where only that length could say where the data ends: with no `endstream` before the next object or the end of the file, the section counts as cut, as above; when its rows spell one, the integer the cut left of that object — whether the section's own row places it, or a newer section does — is where an `endstream` is looked for, the data taken to it when one confirms it, and otherwise the rows past the spelled one the limit's, the section counting as cut there, as above ([#224]). | Each section of the chain | | `xref.section-not-found` | Error | A section a `/Prev` or an `/XRefStm` names is neither there nor within 512 bytes of it, or what names it is not an offset — tiff2pdf writes `/Prev 576066 0 R`. The objects only it indexes are missing from the index. The section `startxref` names is `file.startxref-wrong`'s. A `/Prev` or an `/XRefStm` the reader follows, written as a reference that leads to an object `MaxObjectLength` cuts, names no section missing: the chain ends there for the limit, what it names unread, which `xref.checked-in-part` says ([#263]). | Each section of the chain | | `xref.section-shifted` | Warning | A section a `/Prev` or an `/XRefStm` names starts a few bytes from where it is named, and the reader found it there. | Each section of the chain | | `xref.chain-loop` | Error | The chain names a section it has already read, or an offset it has already named where it read none, in the file or outside it: the reader stops there, and the section the chain should have gone on to is unknown, what only it indexed found, if at all, by rebuilding the index — as for `xref.section-not-found`. | The chain | diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index c02c4a33..b3f0c16c 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3810,12 +3810,13 @@ private bool ReadSubsection(ref PdfLexer lexer, int first, int count, XRefSectio /// /// An object that declares itself a cross-reference stream — /Type /XRef, or a /W, beside no other /// /Type at a place the section may have been moved to — and cannot be read is a malformed section; any - /// other object there, or none, is no section at all, and a guard that cut what declares another type, its dictionary - /// or a value of it, cut no section. What makes it unreadable whatever a value a guard cut holds is the file's fault, - /// as read whole (). At a place the section may have been moved to - /// (), nothing such an object's reading met is reported or recorded — its search for an - /// endstream aside, which is kept as every search is —: it was read as a section, through a section's window, - /// and is read as itself when something asks for it. + /// other object there, or none, is no section at all, and a guard that cut the dictionary of what declares another + /// type cut no section, though one that cut a value of a stream there leaves it the section, as read whole. What + /// makes it unreadable whatever a value a guard cut holds is the file's fault, as read whole + /// (). At a place the section may have been moved to (), + /// nothing such an object's reading met is reported or recorded — its search for an endstream aside, which is + /// kept as every search is —: it was read as a section, through a section's window, and is read as itself when + /// something asks for it. /// /// /// ISO 32000-1 (7.5.8.2) makes the values of this dictionary direct, the elements of /W and /Index @@ -3886,8 +3887,9 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect section.TrailerPosition = absolute; section.TypedXRef = dictionary.IsOfType(PdfName.XRef); - // A guard that cut what declares another type — a font's /W gives widths — cut no section: what is there is the - // file's to be, as read whole (#263). + // A guard that cut the dictionary of what declares another type cut no section: such a dictionary — a CIDFont's, + // whose /W gives widths — has no stream data read whole, and is no section. A stream of that type, which the cut + // hides, would be read as the section whole: the cut takes it for none (#263). section.CutByLimit |= _guardsReached != guards && DeclaresOnlyCrossReferenceStream(value); if (value is not PdfStream xrefStream) @@ -3930,7 +3932,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect ? unplaced : FaultBesideCut(dictionary); section.Fault = file ?? lengthFault; - return file is null ? CutByValue(section, value, past) : XRefSectionState.Malformed; + return file is null ? CutByValue(section, past) : XRefSectionState.Malformed; } var where = search.NextObject is { } next @@ -3959,7 +3961,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect return XRefSectionState.Malformed; } - return CutByValue(section, value, cut); + return CutByValue(section, cut); } var widths = dictionary.GetArray(PdfName.W); @@ -4400,21 +4402,16 @@ private static bool DeclaresCrossReferenceStream(PdfObject value) => /// /// Takes a section a value's cut keeps from being read for the guard's: the section counts as cut, the chain ends there /// and nothing is rebuilt for it (ADR 34), and the object the guard cut is loaded once the chain is read, which reaches - /// the guard (#263). What declares another type is no section a guard cut, a font's /W giving widths: it is the - /// file's to be, as read whole. + /// the guard (#263). Whatever /Type the stream gives, as where the chain names it a stream with a /W is read + /// as the section whole: the value hides whether its rows are. /// /// The section, whose fault names the value. - /// What the offset holds. /// The object the guard cut that the value leads to. - private XRefSectionState CutByValue(XRefSectionRecord section, PdfObject value, PdfObjectId cut) + private XRefSectionState CutByValue(XRefSectionRecord section, PdfObjectId cut) { - if (DeclaresOnlyCrossReferenceStream(value)) - { - section.CutByLimit = true; - section.CutByValue = true; - (_chainValuesCut ??= []).Add(cut); - } - + section.CutByLimit = true; + section.CutByValue = true; + (_chainValuesCut ??= []).Add(cut); return XRefSectionState.Malformed; } @@ -4484,9 +4481,12 @@ private XRefSectionState CutByValue(XRefSectionRecord section, PdfObject value, for (var i = 0; i < ranges.Count; i += 2) { - if (Follow(PdfName.Index, ranges[i], entry: false, out var start, out _) is null && - Follow(PdfName.Index, ranges[i + 1], entry: false, out var count, out _) is null && - SubsectionFault(ranges, i, start?.AsInteger(), count?.AsInteger()) is { } fault) + var startRead = Follow(PdfName.Index, ranges[i], entry: false, out var start, out _) is null; + var countRead = Follow(PdfName.Index, ranges[i + 1], entry: false, out var count, out _) is null; + + if ((startRead && countRead + ? SubsectionFault(ranges, i, start?.AsInteger(), count?.AsInteger()) + : HalfSubsectionFault(ranges, i, startRead, start?.AsInteger(), countRead, count?.AsInteger())) is { } fault) { return fault; } @@ -4495,6 +4495,43 @@ private XRefSectionState CutByValue(XRefSectionRecord section, PdfObject value, return null; } + /// + /// Says what is wrong with the subsection an /Index gives at , one of whose two elements a + /// guard cut, whatever that element holds: the other one is no integer, or a count below 0 or past what object numbers + /// hold, or a first number outside them. Null when the element read is right for some value of the other. + /// + /// The /Index, whose written elements a fault describes. + /// Where the subsection's first number lies. + /// Whether the first number was read. + /// The first number, when it was read and is an integer. + /// Whether the count was read. + /// The count, when it was read and is an integer. + private static string? HalfSubsectionFault(PdfArray ranges, int at, bool startRead, long? start, bool countRead, long? count) + { + if ((startRead && start is null) || (countRead && count is null)) + { + return $"its /Index holds {DescribeValue(ranges[startRead && start is null ? at : at + 1])} where an integer belongs"; + } + + if (countRead && count < 0) + { + return "its /Index gives a subsection a count of rows out of range"; + } + + if (countRead && count > PdfObjectId.MaxNumber + 1L) + { + return string.Create( + CultureInfo.InvariantCulture, + $"its /Index gives a subsection of {count:N0} rows, more than object numbers 0 to {PdfObjectId.MaxNumber} hold"); + } + + return startRead && start is < 0 or > PdfObjectId.MaxNumber + ? string.Create( + CultureInfo.InvariantCulture, + $"its /Index gives a subsection from object {start}, outside object numbers 0 to {PdfObjectId.MaxNumber}") + : null; + } + /// The fault of a /W that is no array of three widths. private const string WidthsMissing = "its /W does not give the widths of three fields"; diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 04d1fedf..e9489ae7 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -2592,15 +2592,23 @@ public void A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_n [InlineData("/Type /XRef /Size 8 /W [1 4] /Length 5 0 R", "Length", null)] [InlineData("/Type /XRef /W [1 4 2] /Index [0] /Length 5 0 R", "Length", null)] [InlineData("/Type /XRef /Size 8 /W 9 0 R /Length 5 0 R", "Length", null)] + [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [5 0 R -1]", "Index element", null)] + [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [5 0 R /x]", "Index element", null)] + [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [/x 5 0 R]", "Index element", null)] + [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [5 0 R 3000000000]", "Index element", null, false)] + [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [-1 5 0 R]", "Index element", null, false)] + [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [9 0 R -1] /Length 5 0 R", "Length", "cut:0")] public void What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_guard_cut_holds_is_the_file_s( - string entries, string cut, string? nine) + string entries, string cut, string? nine, bool asReadWhole = true) { // Stream 7's /Filter, an /Index element, its /W or its /Length names object 5, which MaxObjectLength cuts — the length // where no endstream ends the data. Its /W gives no three widths, or none past 0; its /Index holds no pairs, or a // count of rows below 0, and its /Size no count, written direct or through object 9; or a value past the cut names // object 9, which nothing places: no value the guard hid makes the section readable, and read whole it is the // file's fault. Only a numbering written direct was judged: the section was the guard's, the chain ended, the index - // was not rebuilt and the catalog was lost (#263). + // was not rebuilt and the catalog was lost (#263). Nor was a subsection one of whose elements the guard cut, though + // the other is wrong whatever the cut one holds: no integer, a count below 0 or past what object numbers hold, a + // first number outside them — said as read whole where the cut element is in no part of what is said. var (five, compressed) = cut switch { "Filter" => ("/FlateDecode" + new string(' ', 1100), true), @@ -2614,7 +2622,9 @@ public void What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_gu compressed, predicted: false, placed: true, - placedToo: nine is null ? null : new Dictionary { [9] = nine }); + placedToo: nine is null + ? null + : new Dictionary { [9] = nine.StartsWith("cut:", StringComparison.Ordinal) ? nine[4..] + new string(' ', 1100) : nine }); var text = Encoding.Latin1.GetString(built); if (cut == "Length") @@ -2628,13 +2638,49 @@ public void What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_gu using var document = PdfDocument.Open(file, ObjectGuard); var unreadable = whole.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.XRefSectionUnreadable).Which; - document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.XRefSectionUnreadable) - .Which.Message.Should().Be(unreadable.Message); + var guarded = document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.XRefSectionUnreadable).Which; + + if (asReadWhole) + { + guarded.Message.Should().Be(unreadable.Message); + } document.WasRepaired.Should().BeTrue(); document.Reader.Structure.Sections.Should().NotContain(section => section.CutByLimit); document.GetObject(new PdfObjectId(1)).AsDictionary().IsOfType(PdfName.Catalog).Should().BeTrue(); } + [Theory] + [InlineData("/Type /XRef /Size 9 0 R /W [1 4 2] /Index 5 0 R", "[0 8]", "/Index is the reference 5 0 R")] + [InlineData("/Type /XRef /W [1 4 2] /Index 5 0 R", "[0 8]", "/Index is the reference 5 0 R")] + [InlineData("/Type /XRef /Size 8 /W [0 0 5 0 R]", "7", "/W holds the reference 5 0 R")] + [InlineData("/Type /XRef /Size 8 /W [1 5 0 R 2] /Index [0 9 0 R]", "4", "/W holds the reference 5 0 R")] + [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [5 0 R 8]", "0", "/Index holds the reference 5 0 R")] + public void A_section_a_value_s_cut_keeps_from_being_read_is_the_guard_s_where_nothing_read_says_it_is_wrong( + string entries, string five, string named) + { + // Stream 7's /W, a /W element, its /Index or an /Index element names object 5, which MaxObjectLength cuts, and what + // the rest says is right for some value of it: a /Size no section places beside an /Index, which the rows are read + // without; no /Size beside an /Index; widths of 0 beside the one cut; a count that numbers objects; a second value + // cut, object 9, after the first. The section is the guard's, named by the first value it cut (#263). + var file = ChainFiles.UnderAnUpdate( + entries + " /Length {length}", + five + new string(' ', 1100), + compressed: false, + predicted: false, + placed: true, + placedToo: entries.Contains("[0 9 0 R]", StringComparison.Ordinal) + ? new Dictionary { [9] = "8" + new string(' ', 1100) } + : null); + + using var document = PdfDocument.Open(file, ObjectGuard); + var section = document.Reader.Structure.Sections[^1]; + + section.CutByLimit.Should().BeTrue(); + section.Fault.Should().Contain(named); + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + } + [Theory] [InlineData("/W [1 4] /Size 8 /Filter 5 0 R")] [InlineData("/W 5 0 R /Size 8 /Filter 9 0 R")] @@ -2667,13 +2713,15 @@ public void A_place_a_section_may_have_moved_to_that_a_value_past_a_guard_s_cut_ [Theory] [InlineData("<< /Type /Font /W 5 0 R /Size 8 /Length 3 >>\nstream\nabc\nendstream", "[1 4 2]")] [InlineData("<< /Type /Font /W [1 4 2] /Size 8 /Length 5 0 R >>\nstream\nabc\nendstreaX", "3")] - public void A_stream_of_another_type_whose_value_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut( + [InlineData("<< /Type /Xref /W [1 5 0 R 2] /Size 8 /Length 3 >>\nstream\nabc\nendstream", "4")] + public void A_stream_whose_value_a_guard_cut_where_the_chain_names_a_section_is_the_section_the_guard_cut_whatever_its_type( string six, string five) { - // The /Prev names object 6 exactly, a stream that declares /Type /Font, whose /W, or whose /Length where no endstream - // ends its data before object 8, names object 5, which MaxObjectLength cuts: the guard cut no section, the stream - // saying what it is, as when the guard cuts its dictionary. It was taken for the section the guard cut, the chain - // ended there, and the catalog was lost; the section is found farther on (#263). + // The /Prev names object 6 exactly, a stream that declares /Type /Font, or misspells XRef, whose /W, a /W element, or + // whose /Length where no endstream ends its data before object 8, names object 5, which MaxObjectLength cuts. Where + // the chain names it, a stream with a /W is read as the section whole, whatever its type: the value hides whether its + // rows are, and the section is the guard's. It was taken for no section, the section looked for nearby, and a stream + // found there — an older revision's — read in its place (#263). var built = ChainFiles.UnderAnUpdate( "/Type /XRef /Size 8 /W [1 4 2] /Length {length}", five == "3" ? new string(' ', 1100) + five : five + new string(' ', 1100), @@ -2689,11 +2737,32 @@ public void A_stream_of_another_type_whose_value_a_guard_cut_where_the_chain_nam using var document = PdfDocument.Open(file, ObjectGuard); - document.Reader.Structure.Sections[^1].Offset.Should().Be(stream); - document.Reader.Structure.Sections.Should().NotContain(section => section.CutByLimit); - document.GetObject(new PdfObjectId(1)).AsDictionary().IsOfType(PdfName.Catalog).Should().BeTrue(); + var section = document.Reader.Structure.Sections[^1]; + section.Offset.Should().Be(sixAt); + section.CutByLimit.Should().BeTrue(); + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + } + + [Fact] + public void A_cross_reference_stream_that_misspells_its_type_and_whose_value_a_guard_cut_is_the_section_the_guard_cut() + { + // Stream 7 writes /Type /Xref, which read whole is no fault of the reader's to report, and a /W element that names + // object 5, which MaxObjectLength cuts. It was refused for the type it gives, the file charged with a section it could + // not read, and the index rebuilt (#263). + var file = ChainFiles.UnderAnUpdate( + "/Type /Xref /Size 8 /W [1 5 0 R 2] /Length {length}", "4" + new string(' ', 1100), compressed: false, predicted: false, placed: true); + + using (var whole = PdfDocument.Open(file)) + { + whole.Diagnostics.Should().BeEmpty(); + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.Reader.Structure.Sections[^1].CutByLimit.Should().BeTrue(); document.WasRepaired.Should().BeFalse(); - document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.XRefOffsetAdjusted); } [Fact] From 3f3459a3cb2dac24749312ce603db2771df231f6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 20:13:27 +0000 Subject: [PATCH 120/146] fix: undo as far as it goes a predicted row a hidden end leaves short Data a proven endstream ends past white space under a hidden /Length decodes as read whole, but a filter with no checksum after its last byte, raw deflate or LZW without its end code, runs a byte short where the search left a carriage return out. Its last predicted row was then dropped, left predicted, or reported as rows longer than the data, though the end is the guard's to hide. Such a row is undone as far as it goes, as of data a guard cut. A test holds the LZW end report the hidden end silences. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../IO/Filters/PdfFilterPipeline.cs | 8 +- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 99 +++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs index e0d9253e..9687e2b9 100644 --- a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs +++ b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs @@ -291,7 +291,10 @@ private static ReadOnlyMemory ApplyOne( diagnostics, position); Reached(limited, name, bounds.Shared, diagnostics, position, guard, ref cut, ref limitReported); - return ApplyPredictor(decoded, step, diagnostics, position, cut); + + // Data that ran out where a guard hides its end may have lost the end of its last row there: it is undone as far + // as it goes, as of data a guard cut. + return ApplyPredictor(decoded, step, diagnostics, position, cut || (silenced && ending == FlateEnding.TailLost)); } if (name == PdfName.LZWDecode) @@ -326,7 +329,8 @@ private static ReadOnlyMemory ApplyOne( } Reached(limited, name, bounds.Shared, diagnostics, position, guard, ref cut, ref limitReported); - return ApplyPredictor(decoded, step, diagnostics, position, cut); + return ApplyPredictor( + decoded, step, diagnostics, position, cut || (silenced && ending is DataEnding.Unmarked or DataEnding.UnmarkedInsideCode)); } byte[] text; diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index e9489ae7..213995bd 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1013,6 +1013,105 @@ private static string Ascii85Text(ReadOnlySpan bytes) return text.ToString(); } + [Theory] + [InlineData("cut", 12)] + [InlineData("left out", 12)] + [InlineData("cut", 2)] + public void Predicted_rows_a_hidden_end_leaves_a_byte_short_are_undone_as_far_as_they_go(string guard, int predictor) + { + // Content stream 4's Flate data, raw deflate with no zlib header nor checksum, stores two predicted rows of 40 bytes + // whose last byte is 0x0D, then an end-of-line and its endstream, which the next object proves its own. The search + // leaves the carriage return out with the line feed, which only the hidden length says is the data's or not: the + // stored block runs out a byte short, which says nothing of the file, and the row it parts is the guard's, undone as + // far as it goes. It was dropped, or left predicted (#224). + var content = new List(); + + for (var row = 0; row < 2; row++) + { + var bytes = Encoding.ASCII.GetBytes(Invariant($"Row {row} of predicted data, padded to forty.")[..40]); + bytes[^1] = 0x0D; + + if (predictor != 2) + { + content.Add((byte)(row == 0 ? 0 : 2)); + } + + content.AddRange(bytes); + } + + var length = content.Count; + byte[] raw = [0x01, (byte)length, (byte)(length >> 8), (byte)~length, (byte)(~length >> 8), .. content]; + var data = Encoding.Latin1.GetString(raw); + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, Page), + [20] = file.Object(20, (guard == "cut" ? new string(' ', 2000) : string.Empty) + raw.Length.ToString(CultureInfo.InvariantCulture)), + [4] = file.Object(4, Invariant($"<< /Length 20 0 R /Filter /FlateDecode /DecodeParms << /Predictor {predictor} /Columns 40 >> >>\nstream\n{data}\nendstream")), + [5] = file.Object(5, "(five)"), + }; + file.Table(21, rows, "/Size 21 /Root 1 0 R"); + var limits = guard == "cut" + ? PdfReaderLimits.Default with { MaxObjectLength = 1024 } + : PdfReaderLimits.Default with { MaxIndexEntries = 20 }; + byte[] whole; + + using (var document = PdfDocument.Open(file.Bytes)) + { + whole = document.GetObject(new PdfObjectId(4)).AsStream().Required().Decode().ToArray(); + whole.Should().HaveCount(80); + } + + using var guarded = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = limits }); + var stream = guarded.GetObject(new PdfObjectId(4)).AsStream().Required(); + + stream.Decode().ToArray().Should().Equal(whole[..^1]); + stream.Data.EndHidden.Should().BeTrue(); + guarded.Diagnostics.Where(entry => entry.Code == PdfDiagnosticCodes.FilterFailed).Select(entry => entry.Message) + .Should().Equal("A Flate stream was not valid zlib data."); + } + + [Theory] + [InlineData("cut")] + [InlineData("left out")] + public void Lzw_data_that_lacks_its_end_code_before_a_proven_endstream_past_an_end_of_line_is_no_fault_of_the_file(string guard) + { + // Content stream 4's LZW data, the codes 256 and 113, "q", lacks its end-of-data code, then an end-of-line and its + // endstream, which the next object proves its own: whether the end-of-line is the data's only the hidden length says, + // and that the codes run out there says nothing of the file (#224). + var data = Encoding.Latin1.GetString([0x80, 0x1C, 0x40]); + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, Page), + [20] = file.Object(20, (guard == "cut" ? new string(' ', 2000) : string.Empty) + "3"), + [4] = file.Object(4, $"<< /Length 20 0 R /Filter /LZWDecode >>\nstream\n{data}\nendstream"), + [5] = file.Object(5, "(five)"), + }; + file.Table(21, rows, "/Size 21 /Root 1 0 R"); + var limits = guard == "cut" + ? PdfReaderLimits.Default with { MaxObjectLength = 1024 } + : PdfReaderLimits.Default with { MaxIndexEntries = 20 }; + + using (var whole = PdfDocument.Open(file.Bytes)) + { + whole.GetObject(new PdfObjectId(4)).AsStream().Required().Decode().ToArray().Should().Equal("q"u8.ToArray()); + whole.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.FilterFailed); + } + + using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = limits }); + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + stream.Decode().ToArray().Should().Equal("q"u8.ToArray()); + stream.Data.EndHidden.Should().BeTrue(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal( + guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries); + } + private static byte[] ZLibDecompress(byte[] zlib) { using var input = new MemoryStream(zlib); From 82c1f030bd1576d889c40b1eedb9cc99081126eb Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 20:14:58 +0000 Subject: [PATCH 121/146] fix: read no more rows from data decoded again knowing its end hidden than before A group a filter drops where a hidden end parts it counts against what the filters before the last share, but the filter after it, given less, wrote less, and left the next one more room than the first decoding of the same data gave it. Through four filters, the deferred twin read rows past where the direct twin stopped at MaxDecodedStreamLength: the page indexed beside a guard, or rows and no rebuild where the direct twin rebuilds. The data decoded again holds no more rows than the decoding before it read. Refs #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 13 ++++++ tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 41 ++++++++++++------- .../DocumentReaderTests.cs | 30 ++++++++++++++ 3 files changed, 70 insertions(+), 14 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index b3f0c16c..383e83bd 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -4046,6 +4046,9 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // hidden keeps reached: the whole data holds at least what that decoding read. var reachedBefore = false; + // How many rows the decoding before held, which the data decoded again knowing its end hidden holds no more of. + long heldBefore = 0; + while (true) { var decoded = Filters.PdfFilterPipeline.Decode(xrefStream, _diagnostics, endHidden, reachedBefore, out var leftEncoded, out var cut); @@ -4068,6 +4071,15 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect row = rowLength <= data.Length ? (int)rowLength : 0; held = row == 0 ? 0 : data.Length / row; + // The data decoded again knowing its end hidden lacks the group a filter dropped there, and a filter after it, + // given less, may be given more room than the decoding before gave it, and read rows past where that one stopped + // at a guard on decoding: the whole data holds no more of them than that decoding read (#220). + if (endHidden && held > heldBefore) + { + held = heldBefore; + data = data[..(int)(held * row)]; + } + // Fewer rows than the section declares, in data a /Length the chain could not read left to the first endstream: // where the section's own row for that /Length, among those held, gives more data, which an endstream confirms, // the rows spelled an endstream, and the data is taken again to that length (#220). Data that holds no row whole @@ -4102,6 +4114,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect (lengthCut || (!retaken && !decodedWhole && declared > held))) { reachedBefore = cut; + heldBefore = held; if (reachedBefore) { diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 502ee873..9bc21f68 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -1236,7 +1236,15 @@ public static byte[] UpdateHidingItsHighestNumber(int padding) /// otherwise /Filter [/ASCII85Decode /FlateDecode /RunLengthDecode], the last ASCII85 group partial. RunLength /// writes literal runs of 128 bytes. /// - public static byte[] RowsThroughThreeFilters(bool deferred, bool hex) + public static byte[] RowsThroughThreeFilters(bool deferred, bool hex) => + RowsThroughFilters(deferred, hex ? ["AHx", "RL", "Fl"] : ["A85", "Fl", "RL"]); + + /// + /// A file as writes it, its rows passing through , as + /// /Filter names them: A85, AHx, Fl or RL. The first is ASCII85, its last group + /// partial and no ~>, or ASCIIHex short of its last digit and its >. + /// + public static byte[] RowsThroughFilters(bool deferred, string[] filters) { using var file = new Writer("1.5"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); @@ -1251,25 +1259,30 @@ public static byte[] RowsThroughThreeFilters(bool deferred, bool hex) rows[4] = (1, four, 0); rows[5] = (1, at, 0); rows[39] = (1, file.OffsetOf(39), 0); - var raw = Encode(rows, Rows.Raw); - byte[] data; + var data = Encode(rows, Rows.Raw); - if (hex) - { - var digits = Convert.ToHexString(RunLength(Deflate(raw))); - data = Encoding.ASCII.GetBytes(digits[..^1]); - } - else + for (var i = filters.Length - 1; i >= 0; i--) { - data = Encoding.ASCII.GetBytes(Ascii85(Deflate(RunLength(raw)))); + data = filters[i] switch + { + "A85" => Encoding.ASCII.GetBytes(Ascii85(data)), + "AHx" => Encoding.ASCII.GetBytes(Convert.ToHexString(data)[..^1]), + "Fl" => Deflate(data), + "RL" => RunLength(data), + _ => throw new ArgumentOutOfRangeException(nameof(filters), filters[i], "No such filter."), + }; } + var names = string.Join(' ', Array.ConvertAll(filters, filter => filter switch + { + "A85" => "/ASCII85Decode", + "AHx" => "/ASCIIHexDecode", + "Fl" => "/FlateDecode", + _ => "/RunLengthDecode", + })); var length = data.Length.ToString("D6", CultureInfo.InvariantCulture); file.Object(4, length); - file.Stream( - 5, - $"/Type /XRef /Size 40 /W [1 4 2] /Filter {(hex ? "[/ASCIIHexDecode /RunLengthDecode /FlateDecode]" : "[/ASCII85Decode /FlateDecode /RunLengthDecode]")} /Root 1 0 R /Length {(deferred ? "4 0 R" : length)}", - data); + file.Stream(5, $"/Type /XRef /Size 40 /W [1 4 2] /Filter [{names}] /Root 1 0 R /Length {(deferred ? "4 0 R" : length)}", data); file.StartXRef(at); return file.ToArray(); diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 53c2d03d..617c73d0 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -2005,6 +2005,36 @@ public void A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedSt deferred.Reader.Structure.Sections[^1].Incomplete.Should().BeFalse(); } + [Theory] + [InlineData("A85 RL RL RL")] + [InlineData("A85 RL RL Fl")] + [InlineData("A85 RL Fl RL")] + public void Data_whose_end_a_guard_hides_holds_no_more_rows_than_its_direct_twin_through_four_filters(string chain) + { + // As below, through four filters: the second, given less, wrote less, and left the third more room than the direct + // twin's decoding did, which read on, and the last read rows past where the direct twin stopped at the bound — the + // page indexed beside a guard, or rows and no rebuild where the direct twin rebuilds (#220). + var direct = ChainFiles.RowsThroughFilters(deferred: false, chain.Split(' ')); + var deferred = ChainFiles.RowsThroughFilters(deferred: true, chain.Split(' ')); + + for (var bound = 1; bound <= 1500; bound++) + { + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxDecodedStreamLength = bound } }; + using var directDocument = PdfDocument.Open(direct, options); + using var deferredDocument = PdfDocument.Open(deferred, options); + + Read(deferredDocument).Should().Be(Read(directDocument), $"MaxDecodedStreamLength is {bound}"); + } + + static string Read(PdfDocument document) => string.Join( + ' ', + document.WasRepaired, + document.Reader.Structure.Sections[^1].CutByLimit, + document.GetObject(new PdfObjectId(1)) is PdfDictionary, + document.GetObject(new PdfObjectId(39)) is PdfDictionary, + document.Reader.ChainIndex?.Count); + } + [Theory] [InlineData(false)] [InlineData(true)] From 63e185b47a92761b4c93c86de507a1b3e06210f6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 20:19:13 +0000 Subject: [PATCH 122/146] fix: read on once past an endstream however many places lead to it The read-on past an endstream, which tells whether endobj follows it, was kept for each place a length ends at. A cross-reference stream whose /Length the chain cannot read keeps the carriage return of its last row as data, and its length, checked once the chain is read, ends before it: two places before one keyword, each reading on through the white space after it. Thirty such revisions followed by 64 KB of white space each had the searches read close to four times the file, the most they may; they read it about twice now, as with a line feed alone. Refs #174, #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 2 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 37 ++++++++++-------- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 38 +++++++++++++++++++ .../DocumentReaderTests.cs | 16 ++++++++ 4 files changed, 77 insertions(+), 16 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index 97a2175b..85f9b6d7 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -356,7 +356,7 @@ Known gaps: | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | | A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`), so that a filter that runs out there — a Flate checksum short of a carriage return, an end marker missing — says nothing of the file, while the data decodes as read whole: white space parts no ASCII85 or ASCIIHex group, and no group is dropped nor a predictor's row undone for it. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `A_cross_reference_stream_named_before_white_space_proves_the_endstream_before_it_however_much`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `Data_that_ends_in_white_space_right_before_a_proven_endstream_ends_where_the_guard_hides`, `An_ascii85_object_stream_that_lacks_its_end_marker_before_a_proven_endstream_decodes_as_read_whole`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | -| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, and the white space it crosses once more at most — the run before an `endobj` taken from the read-on that read it, a run a row names before the next header read once from the row —, about three times the file in all; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_proofs_of_streams_white_space_follows_read_no_stretch_of_the_file_more_than_three_times`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | +| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, and the white space it crosses once more at most — the run before an `endobj` taken from the read-on that read it, a run a row names before the next header read once from the row —, about three times the file in all; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `DocumentReaderTests.Deferred_lengths_followed_by_long_white_space_have_it_read_on_once_however_their_data_ends`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_proofs_of_streams_white_space_follows_read_no_stretch_of_the_file_more_than_three_times`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | | Encoded data of any length: a `/Length` of a few digits claiming gigabytes, one past what an array holds, which would exhaust memory at each read, or data decoding hands back as it is, which `MaxDecodedStreamLength` does not bound | `MaxStreamLength` (*The guards*): what is read of one stream's data at once stops at it, what lies past it is cut by a guard and reported as `limit.stream-length` — once for the stream in its document's diagnostics, and with each decoding that meets the cut in diagnostics a caller passes to `Decode`, as `limit.decoded-stream` is —, and the filters decode what was read as data the reader cut ([#170]) | `ReaderLimitsTests.A_stream_cut_at_MaxStreamLength_decodes_what_lies_within_it_as_data_the_reader_cut_and_is_reported_once`, `Data_decoding_hands_back_as_it_is_is_bounded_by_MaxStreamLength_and_not_by_MaxDecodedStreamLength`, `A_length_past_what_an_array_holds_ends_at_the_guard_and_never_in_an_exhausted_memory`, `Decoding_into_diagnostics_of_one_s_own_reports_the_cut_max_stream_length_made_each_time` | No | diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 383e83bd..63a6433a 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -149,12 +149,14 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// /Length the chain cannot read (#220) is searched once to its endstream; a stream whose /// /Length a guard cut or left out of the index (#224) is searched once to its endstream, or to a /// header its data spells, which leaves the data up to it to the guard; and white space after a declared length - /// that runs past a first read is read on once for each place (#174). The data of different streams lies apart, and so do the runs of white space - /// after them: together those reads cover the file once, and one read past each — about twice the file at most. The - /// fourth adds a little to each stream of the second way: the proof that an endstream found is the stream's - /// own (#224) reads past it to its object's end and what follows, three first reads of - /// bytes, and a fourth past a row that names the white space before the next object's header, each once however - /// often the stream is parsed — 48 or 64 bytes, about what the smallest such stream spans with its header. White + /// that runs past a first read is read on once for each place, and past an endstream once for each keyword + /// however many places lead to it (#174). The data of different streams lies apart, and so do the runs of white + /// space after them: together those reads cover the file once, and one read past each — about twice the file at + /// most. The fourth adds a little to each stream of the second way: the proof that an endstream found is the + /// stream's own (#224) reads past it to its object's end and what follows, three first reads of + /// bytes, and a fourth past a row that names the white space before the next object's + /// header, each once however often the stream is parsed — 48 or 64 bytes, about what the smallest such stream spans + /// with its header. White /// space it crosses adds to that: the run before the object's endobj it takes from the read-on that read it, /// the run after it it doubles through, as the read-on does, and the run a row names before the next header it /// reads once from there, a first read reaching the header. No run is read more than three times, and the file @@ -404,9 +406,9 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider private readonly Dictionary _endStreamChecks = []; /// - /// Where the first significant byte lies past each endstream keyword the reader read on past (#174), which an - /// endobj there starts at: the proof that the keyword ends the stream's own data (#224) takes it rather than read - /// the white space again. + /// Where the first significant byte lies past each endstream keyword the reader read on past (#174), an + /// endobj there or not: another place that leads to the keyword, and the proof that the keyword ends the stream's + /// own data (#224), take it rather than read the white space again. /// private readonly Dictionary _endObjStarts = []; @@ -1728,15 +1730,20 @@ long IPdfStreamDataProvider.DataStart(long absoluteKeywordEnd) keywordEnd = at + StreamEnds.EndStreamKeyword.Length; } - var next = NextSignificant(keywordEnd); - state = next < 0 ? EndObjState.Unseen : IsEndObjKeywordAt(next) ? EndObjState.Present : EndObjState.Absent; - _endStreamChecks[absoluteOffset] = state; - - if (state == EndObjState.Present) + // Two places may lead to one keyword — a cross-reference stream's data, whose carriage return is kept, and the + // length that ends before it —: what follows the keyword is read once. + if (!_endObjStarts.TryGetValue(keywordEnd, out var next)) { - _endObjStarts[keywordEnd] = next; + next = NextSignificant(keywordEnd); + + if (next >= 0) + { + _endObjStarts[keywordEnd] = next; + } } + state = next < 0 ? EndObjState.Unseen : IsEndObjKeywordAt(next) ? EndObjState.Present : EndObjState.Absent; + _endStreamChecks[absoluteOffset] = state; return state; } diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 9bc21f68..f9a67280 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -1314,6 +1314,42 @@ static byte[] RunLength(byte[] bytes) } } + /// + /// A file of revisions, each a cross-reference stream whose /Length names an object only + /// its own rows place, its rows ending in before endstream, then + /// spaces before endobj: valid, its searches reading on through each run. + /// + public static byte[] UpdatesWithDeferredLengthsBeforeWhiteSpace(int count, int run, string endOfLine) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + var previous = -1L; + + for (var i = 0; i < count; i++) + { + var length = 4 + (2 * i); + var stream = length + 1; + file.Object(length, Invariant($"{(i == 0 ? 6 : 2) * 7}")); + var at = file.Position; + (int Type, long Second, int Third)[] own = [(1, file.OffsetOf(length), 0), (1, at, 0)]; + var rows = i == 0 + ? Encode([(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), .. own], Rows.Raw) + : Encode(own, Rows.Raw); + file.Text(Invariant( + $"{stream} 0 obj\n<< /Type /XRef /Size {stream + 1} /W [1 4 2] /Index [{(i == 0 ? "0 6" : $"{length} 2")}] /Root 1 0 R{(previous < 0 ? string.Empty : $" /Prev {previous}")} /Length {length} 0 R >>\nstream\r\n")); + file.Bytes(rows); + file.Text(endOfLine + "endstream" + new string(' ', run) + "endobj\n"); + file.StartXRef(at); + previous = at; + } + + return file.ToArray(); + + static string Invariant(FormattableString text) => text.ToString(CultureInfo.InvariantCulture); + } + private enum Rows { Raw, @@ -1394,6 +1430,8 @@ private sealed class Writer : IDisposable public void Text(string text) => _output.Write(Encoding.Latin1.GetBytes(text)); + public void Bytes(byte[] bytes) => _output.Write(bytes); + public void Object(int number, string body) { _offsets[number] = Position; diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 617c73d0..bce88c64 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -2005,6 +2005,22 @@ public void A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedSt deferred.Reader.Structure.Sections[^1].Incomplete.Should().BeFalse(); } + [Theory] + [InlineData("\n")] + [InlineData("\r\n")] + public void Deferred_lengths_followed_by_long_white_space_have_it_read_on_once_however_their_data_ends(string endOfLine) + { + // Thirty revisions, each a cross-reference stream whose /Length only its own rows place, its endstream followed by + // 65,472 spaces before endobj. With a carriage return before the line feed, which a deferred length's data keeps, the + // search's end and the length's lie apart before one keyword, and each read on through the spaces: the file's + // searches read close to four times its length, the most they may, where twice is said (#174, #220). + var file = ChainFiles.UpdatesWithDeferredLengthsBeforeWhiteSpace(30, 65_472, endOfLine); + using var document = PdfDocument.Open(file); + + document.Diagnostics.Should().BeEmpty(); + document.Reader.EndStreamSearchBytes.Should().BeLessThan(2L * file.Length + 1024); + } + [Theory] [InlineData("A85 RL RL RL")] [InlineData("A85 RL RL Fl")] From 178e6dbf5dadcc0285dcf86ae989d689724df9b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 20:21:03 +0000 Subject: [PATCH 123/146] docs: say what a hidden end and a value's cut leave to the file, and what decodes again The concept page counted two ways a valid file is read past the parser's bytes, about twice the file, where four come to about three times it. The architecture said data whose end a guard hides drops a group, which data an endstream proven its own ends no longer does. ADR 34, the architecture and the concepts said a section a value's cut keeps from being read always ends the chain, with no word of the faults the values read charge to the file. The filters' and DataFaults' docs named data the reader cut alone where the caller's hidden end counts too, and a proof-cost test's name claimed a bound per run it does not assert. Refs #224, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- ...alid-pdf-is-readable-and-the-readers-guards-are.md | 3 ++- docs/architecture.md | 11 +++++++---- docs/threat-model.md | 8 ++++---- docs/website/docs/concepts/reader-limits.md | 9 ++++++--- src/AdCodicem.Pdf/IO/Filters/Ascii85Filter.cs | 4 ++-- src/AdCodicem.Pdf/IO/Filters/AsciiHexFilter.cs | 4 ++-- src/AdCodicem.Pdf/IO/Filters/DataFaults.cs | 4 ++-- tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 2 +- 8 files changed, 26 insertions(+), 19 deletions(-) diff --git a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md index e5f28c65..3ec2694f 100644 --- a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md +++ b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md @@ -125,7 +125,8 @@ on 2026-10-24) —, stops the search, the data up to it then the guard's; a whole reference the cut left leads on, as read whole; a chain of filters ends there, its data left encoded; and a cross-reference section whose values lead to an object `MaxObjectLength` cut ends the -chain, nothing rebuilt, the older sections and their trailers the guard's to hide, and the guard reached once the chain +chain, nothing rebuilt — unless what the values read say makes it unreadable whatever the cut hides, the file's fault +(amended on 2026-10-24) —, the older sections and their trailers the guard's to hide, and the guard reached once the chain is read. So does a cross-reference stream's `/Length` the chain could not read that leads to such an object, wherever only that length could say how far the data runs: no `endstream` comes before the next object or the end of the file, or the rows spell one and none confirms what the cut left of the length, the rows past it then the guard's. A guard that diff --git a/docs/architecture.md b/docs/architecture.md index ec641916..0ee4016e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -141,16 +141,19 @@ separate types, separate severity scales, and vocabularies that never share a co spells or an `endstream` nothing after it proves the stream's own (item 4); a reference the cut left whole is read through, as a load reads it; its chain of filters ends there, with the data left encoded; and the chain of cross-reference sections ends at that section, with nothing rebuilt and the - older sections and their trailers the guard's to hide, wherever near its named offset the section is found. An + older sections and their trailers the guard's to hide, wherever near its named offset the section is found — + unless what the values read say makes it unreadable whatever the cut hides, which is the file's fault. An object `MaxObjectLength` cut there is loaded once the chain is read, which reaches the guard — thrown at opening under `ThrowOnLimit` —, for a value the chain keeps: not for a place it dropped as no section. A cross-reference stream's `/Length` the chain could not read, which leads to such an object, ends the chain so wherever only it could say how far the data runs: with no `endstream` before the next object or the end of the file, the section is the guard's, not malformed; where the rows spell one and no `endstream` confirms what the cut left of the length, the rows past it are the guard's — unless what follows the `endstream` the data was taken to - proves it the stream's own, as in item 4, which leaves the rows the data lacks to the file. Data whose end a - guard hides is decoded as data a guard cut, a group its end parts dropped and what a filter says of the end - silenced; a decoding guard the first decoding reached stands. + proves it the stream's own, as in item 4, which leaves the rows the data lacks to the file. Such data, whose end + a guard hides, is decoded again as data a guard cut, a group its end parts dropped and what a filter says of the + end silenced, no more rows held than the first decoding read; a decoding guard the first decoding reached stands. + Data an `endstream` proven its own ends past white space (item 4) decodes as read whole, what a filter says of + its end silenced. ### 3.2 The write path diff --git a/docs/threat-model.md b/docs/threat-model.md index 85f9b6d7..31460093 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -284,7 +284,7 @@ Known gaps: | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). A hybrid file's stream's own `/Prev`, which the chain does not follow, going on through the table's, is not read at all, nor are the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place: neither names a section missing nor ends the chain for a guard, and the rule on trailer values judges them as written, a reference they hold not followed ([#263]). An object the guard cut after a whole reference it holds hides nothing: the chain reads through it, as a load does, and loads it once the chain is read. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s`, `A_table_an_xrefstm_names_in_a_stream_s_place_has_its_own_links_left_unread`, `A_prev_written_as_a_reference_to_a_real_is_judged_through_it_only_where_the_reader_follows_it` | Yes | | What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); so is one that declared itself a section before a guard cut it — a table's `xref` keyword, `/Type /XRef`. A place that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is a guess dropped, the next place tried; where the chain names a section, a guard that cut the dictionary of what declares another type cut no section — a CIDFont's, which read whole no stream data follows —, and relocation is tried, while one that cut a value of a stream there leaves it the section the guard cut, whatever type it gives, as read whole it is the section. What makes a section unreadable whatever a value a guard cut holds is the file's fault, as read whole, judged before the cut is taken for the guard's (`PdfFileReader.FaultBesideCut`): a value past the cut that no section read so far places, which the sections a whole read reads before make the file's to lack; a `/W` that gives no three widths, or none past 0; rows numbered wrongly — an `/Index` that holds no pairs, whatever its elements, a subsection out of range, or wrong by the element read whatever the cut one holds, a `/Size` with no `/Index` that gives no count —, as far as the values read, written direct or through references, say: a value a guard cut says nothing, nor does what depends on it, the `/Size` an `/Index` so cut may stand beside | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_cross_reference_stream_whose_deferred_length_a_guard_cut_is_found_where_it_was_moved`, `A_cross_reference_stream_a_guard_cut_the_dictionary_of_after_its_type_is_the_section_where_it_was_moved`, `A_classic_table_a_guard_cut_the_rows_of_is_the_section_where_it_was_moved`, `A_stream_with_no_numbering_whose_width_a_guard_cut_is_no_section_the_guard_cut`, `A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild`, `What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_guard_cut_holds_is_the_file_s`, `A_place_a_section_may_have_moved_to_that_a_value_past_a_guard_s_cut_makes_unreadable_is_dropped`, `A_stream_whose_value_a_guard_cut_where_the_chain_names_a_section_is_the_section_the_guard_cut_whatever_its_type`, `A_cross_reference_stream_that_misspells_its_type_and_whose_value_a_guard_cut_is_the_section_the_guard_cut`, `A_section_a_value_s_cut_keeps_from_being_read_is_the_guard_s_where_nothing_read_says_it_is_wrong` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, may stay within the bound — the group counted against what the filters before the last share as well —: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_decoding_guard_deferred_data_reached_is_reported_once_whatever_room_the_diagnostics_have`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `Data_whose_end_a_guard_hides_leaves_the_filters_after_a_dropped_group_the_room_its_direct_twin_s_have`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file`, `A_cross_reference_stream_the_chain_read_proves_the_endstream_of_the_older_one_before_it` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, may stay within the bound — the group counted against what the filters before the last share as well, and no more rows held than the first decoding read —: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_decoding_guard_deferred_data_reached_is_reported_once_whatever_room_the_diagnostics_have`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `Data_whose_end_a_guard_hides_leaves_the_filters_after_a_dropped_group_the_room_its_direct_twin_s_have`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file`, `A_cross_reference_stream_the_chain_read_proves_the_endstream_of_the_older_one_before_it` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, written direct — an integer, or a real with no fractional part; one written as a reference is not read through for it, and bounds nothing —, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | @@ -356,7 +356,7 @@ Known gaps: | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | | A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`), so that a filter that runs out there — a Flate checksum short of a carriage return, an end marker missing — says nothing of the file, while the data decodes as read whole: white space parts no ASCII85 or ASCIIHex group, and no group is dropped nor a predictor's row undone for it. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `A_cross_reference_stream_named_before_white_space_proves_the_endstream_before_it_however_much`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `Data_that_ends_in_white_space_right_before_a_proven_endstream_ends_where_the_guard_hides`, `An_ascii85_object_stream_that_lacks_its_end_marker_before_a_proven_endstream_decodes_as_read_whole`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | -| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, and the white space it crosses once more at most — the run before an `endobj` taken from the read-on that read it, a run a row names before the next header read once from the row —, about three times the file in all; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `DocumentReaderTests.Deferred_lengths_followed_by_long_white_space_have_it_read_on_once_however_their_data_ends`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_proofs_of_streams_white_space_follows_read_no_stretch_of_the_file_more_than_three_times`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | +| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, and the white space it crosses once more at most — the run before an `endobj` taken from the read-on that read it, a run a row names before the next header read once from the row —, about three times the file in all; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `DocumentReaderTests.Deferred_lengths_followed_by_long_white_space_have_it_read_on_once_however_their_data_ends`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_proofs_of_streams_white_space_follows_read_less_than_three_times_the_file`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | | Encoded data of any length: a `/Length` of a few digits claiming gigabytes, one past what an array holds, which would exhaust memory at each read, or data decoding hands back as it is, which `MaxDecodedStreamLength` does not bound | `MaxStreamLength` (*The guards*): what is read of one stream's data at once stops at it, what lies past it is cut by a guard and reported as `limit.stream-length` — once for the stream in its document's diagnostics, and with each decoding that meets the cut in diagnostics a caller passes to `Decode`, as `limit.decoded-stream` is —, and the filters decode what was read as data the reader cut ([#170]) | `ReaderLimitsTests.A_stream_cut_at_MaxStreamLength_decodes_what_lies_within_it_as_data_the_reader_cut_and_is_reported_once`, `Data_decoding_hands_back_as_it_is_is_bounded_by_MaxStreamLength_and_not_by_MaxDecodedStreamLength`, `A_length_past_what_an_array_holds_ends_at_the_guard_and_never_in_an_exhausted_memory`, `Decoding_into_diagnostics_of_one_s_own_reports_the_cut_max_stream_length_made_each_time` | No | @@ -383,8 +383,8 @@ Known gaps: - Accepted, with no issue: under a `/Length` a guard cut or left out of the index, data an `endstream` proven its own ends past white space — the end-of-line the search leaves out before the keyword, above all — decodes with its end the guard's: a Flate stream that lost its tail or its checksum, or ASCII85, ASCIIHex, RunLength or LZW data without - its end marker, is not reported there, though a whole read reports it; only the hidden length says whether that white - space is the data's ([#224]). + its end marker, is not reported there, though a whole read reports it, by any filter of the stream's chain; only the + hidden length says whether that white space is the data's ([#224]). - Accepted, with no issue: where the chain names a section, an object that declares another type and gives a `/W` before `MaxTrailerLength` cuts its dictionary is taken for no section, as a CIDFont's dictionary is read whole, no stream data after it, and the section is looked for nearby; a stream of that type, whose stream data the cut hides, diff --git a/docs/website/docs/concepts/reader-limits.md b/docs/website/docs/concepts/reader-limits.md index de3b7b2f..06463b5b 100644 --- a/docs/website/docs/concepts/reader-limits.md +++ b/docs/website/docs/concepts/reader-limits.md @@ -63,7 +63,8 @@ then the file's, though white space before the keyword, which only the length sa ends to the guard as the data decodes: a filter that runs out there says nothing of the file. Decoding stops at a filter so named, or one whose parameters are, and leaves the rest of the data encoded, as past the most filters the reader runs; and a `/Prev` or an `/XRefStm` the reader follows that is so named, or a cross-reference stream whose values lead to an object a guard cut, ends the chain as a -section a guard cut does, wherever near its named offset the section is found. +section a guard cut does, wherever near its named offset the section is found — unless what the values read say makes +the section unreadable whatever the cut hides: a fault of the file, as read whole. What a guard leaves out of the cross-reference index — the oldest sections of a long chain, the rows past a long table's bound, the objects past the most the index holds — is no damage either. Those objects read as null, unknown @@ -121,8 +122,10 @@ order ends where it breaks it. A sound stream's `endstream` follows its declared the object through a larger window, so that a stream's data — and the end-of-line after its `stream` keyword — never counts toward its object's length. A document's searches read four times the file at most: a valid file is read past those bytes only for the data of a cross-reference -stream whose `/Length` the chain cannot read yet, and for white space and comments after a declared length, about twice -the file. Those stay internal constants, since lifting them would let nothing more be read, and +stream whose `/Length` the chain cannot read yet, for the data of a stream whose `/Length` a guard cut or left out of the +index, and for white space and comments after a declared length, about twice the file, and for the proof that an +`endstream` found under a hidden `/Length` is the stream's own, about three times the file in all. Those stay internal +constants, since lifting them would let nothing more be read, and each says where it is declared why no valid file reaches it. A new bound is classified when it is added ([ADR 34](/project/adr/every-valid-pdf-is-readable-and-the-readers-guards-are)): diff --git a/src/AdCodicem.Pdf/IO/Filters/Ascii85Filter.cs b/src/AdCodicem.Pdf/IO/Filters/Ascii85Filter.cs index 5f795f9d..5f05bb85 100644 --- a/src/AdCodicem.Pdf/IO/Filters/Ascii85Filter.cs +++ b/src/AdCodicem.Pdf/IO/Filters/Ascii85Filter.cs @@ -18,8 +18,8 @@ internal static class Ascii85Filter /// What the data holds that the format does not allow. /// The most the data may decode to. /// - /// Whether the data ends where the file's may not — the reader cut it short at one of its limits, or a guard hides - /// where it ends —: a group left partial at its end, before any ~, is one the cut parted, and is dropped rather + /// Whether the data ends where the file's may not — the reader cut it short at one of its limits, or the caller says a + /// guard hides where it ends —: a group left partial at its end, before any ~, is one the cut parted, and is dropped rather /// than completed, though the bytes it encodes count against , which /// then says, and counts them. /// diff --git a/src/AdCodicem.Pdf/IO/Filters/AsciiHexFilter.cs b/src/AdCodicem.Pdf/IO/Filters/AsciiHexFilter.cs index 251fa9c3..d8781c70 100644 --- a/src/AdCodicem.Pdf/IO/Filters/AsciiHexFilter.cs +++ b/src/AdCodicem.Pdf/IO/Filters/AsciiHexFilter.cs @@ -18,8 +18,8 @@ internal static class AsciiHexFilter /// What the data holds that the format does not allow. /// The most the data may decode to. /// - /// Whether the data ends where the file's may not — the reader cut it short at one of its limits, or a guard hides - /// where it ends —: a digit left alone at its end, before any >, is the first of a pair the cut parted, and is + /// Whether the data ends where the file's may not — the reader cut it short at one of its limits, or the caller says a + /// guard hides where it ends —: a digit left alone at its end, before any >, is the first of a pair the cut parted, and is /// dropped rather than completed with a zero, though the byte counts against , which /// then says, and counts it. /// diff --git a/src/AdCodicem.Pdf/IO/Filters/DataFaults.cs b/src/AdCodicem.Pdf/IO/Filters/DataFaults.cs index 30b143ba..94110013 100644 --- a/src/AdCodicem.Pdf/IO/Filters/DataFaults.cs +++ b/src/AdCodicem.Pdf/IO/Filters/DataFaults.cs @@ -47,8 +47,8 @@ internal struct DataFaults public int UnitHeld; /// - /// For data the reader cut that ends inside an ASCII85 group or an ASCIIHex pair: how many bytes the unit would decode - /// to, which were not written. + /// For data the reader cut, or whose end the caller says a guard hides, that ends inside an ASCII85 group or an + /// ASCIIHex pair: how many bytes the unit would decode to, which were not written. /// public int Dropped; diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 213995bd..2fbbd717 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1490,7 +1490,7 @@ public void The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_ [InlineData(true, false)] [InlineData(false, true)] [InlineData(true, true)] - public void The_proofs_of_streams_white_space_follows_read_no_stretch_of_the_file_more_than_three_times( + public void The_proofs_of_streams_white_space_follows_read_less_than_three_times_the_file( bool beforeEndObj, bool beforeNextHeader) { // Thirty streams whose /Length, object 20, MaxObjectLength cuts end in 65,520 bytes of white space, before their endobj From 9932578454b2f075326145b86880d66a81dd4d56 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 20:29:32 +0000 Subject: [PATCH 124/146] fix: leave to the guard a Flate checksum that hidden white space completes Data a proven endstream ends past white space under a hidden /Length takes in white space the length would leave out: a Flate stream short of its checksum's last byte read a space or a line feed in its place, and reported a checksum that disagrees, a warning, where read whole it is only short. A checksum that ends in that white space says nothing of the file, as one it leaves short. The decoders' docs no longer say such data decodes as read whole everywhere: LZW or RunLength data that lacks its end marker takes the white space in. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/threat-model.md | 4 +- .../IO/Filters/PdfFilterPipeline.cs | 17 ++++--- src/AdCodicem.Pdf/Objects/PdfStreamData.cs | 4 +- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 48 +++++++++++++++++++ 4 files changed, 63 insertions(+), 10 deletions(-) diff --git a/docs/threat-model.md b/docs/threat-model.md index 31460093..d8d4d1b1 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -355,7 +355,7 @@ Known gaps: | A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | -| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`), so that a filter that runs out there — a Flate checksum short of a carriage return, an end marker missing — says nothing of the file, while the data decodes as read whole: white space parts no ASCII85 or ASCIIHex group, and no group is dropped nor a predictor's row undone for it. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `A_cross_reference_stream_named_before_white_space_proves_the_endstream_before_it_however_much`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `Data_that_ends_in_white_space_right_before_a_proven_endstream_ends_where_the_guard_hides`, `An_ascii85_object_stream_that_lacks_its_end_marker_before_a_proven_endstream_decodes_as_read_whole`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | +| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`), so that a filter that runs out there — a Flate checksum short of a carriage return, or one that white space completes, an end marker missing — says nothing of the file, while the data decodes as read whole: white space parts no ASCII85 or ASCIIHex group, and no group is dropped nor a predictor's row undone for it. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `A_cross_reference_stream_named_before_white_space_proves_the_endstream_before_it_however_much`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `Data_that_ends_in_white_space_right_before_a_proven_endstream_ends_where_the_guard_hides`, `A_checksum_short_of_a_byte_white_space_before_a_proven_endstream_completes_is_no_fault_of_the_file`, `Predicted_rows_a_hidden_end_leaves_a_byte_short_are_undone_as_far_as_they_go`, `Lzw_data_that_lacks_its_end_code_before_a_proven_endstream_past_an_end_of_line_is_no_fault_of_the_file`, `An_ascii85_object_stream_that_lacks_its_end_marker_before_a_proven_endstream_decodes_as_read_whole`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | | Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, and the white space it crosses once more at most — the run before an `endobj` taken from the read-on that read it, a run a row names before the next header read once from the row —, about three times the file in all; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `DocumentReaderTests.Deferred_lengths_followed_by_long_white_space_have_it_read_on_once_however_their_data_ends`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_proofs_of_streams_white_space_follows_read_less_than_three_times_the_file`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | @@ -382,7 +382,7 @@ Known gaps: there: only the hidden length could tell, and a valid file's last stream before its table is told so ([#224]). - Accepted, with no issue: under a `/Length` a guard cut or left out of the index, data an `endstream` proven its own ends past white space — the end-of-line the search leaves out before the keyword, above all — decodes with its end - the guard's: a Flate stream that lost its tail or its checksum, or ASCII85, ASCIIHex, RunLength or LZW data without + the guard's: a Flate stream that lost its tail or its checksum, or whose checksum ends in that white space, or ASCII85, ASCIIHex, RunLength or LZW data without its end marker, is not reported there, though a whole read reports it, by any filter of the stream's chain; only the hidden length says whether that white space is the data's ([#224]). - Accepted, with no issue: where the chain names a section, an object that declares another type and gives a `/W` diff --git a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs index 9687e2b9..0f296562 100644 --- a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs +++ b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs @@ -40,8 +40,8 @@ public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diag /// the reader lets data pass through, or a value the filters read that a guard cut or left out of the index: what comes /// back is then the data still encoded by the filters past it, no part of what the stream decodes to —, and whether one /// cut the data, as a reader of its rows or its objects must know. Data a proven endstream ended past white space - /// that only a hidden length says is the data's or not () decodes as read whole, - /// and a filter that runs out at its end says nothing of the file. + /// that only a hidden length says is the data's or not () is not decoded as cut, + /// and a filter that runs out at its end, or whose checksum ends in that white space, says nothing of the file. /// /// The stream to decode. /// Receives what decoding met; when none are supplied, the stream's document does. @@ -162,7 +162,8 @@ private static ReadOnlyMemory DecodeChain( // Data whose endstream is proven its own past white space only a hidden length says is the data's or not ends where // the file's does but for that white space, which parts no ASCII85 or ASCIIHex group and falls in a Flate stream's - // checksum: it decodes as it would read whole, and a filter that runs out there says nothing of the file. + // checksum: no group is dropped and no row undone for it, so that it decodes as read whole, but for LZW or RunLength + // data that lacks its end marker and takes the white space in; a filter that runs out there says nothing of the file. var silenced = stream.Data.EndHidden; // A guard a decoding of the same data reached before was reported then: its verdict stands, whatever this one meets. @@ -240,7 +241,7 @@ public static bool IsImageFilter(PdfName name) => /// /// /// Whether where the data ends says nothing of the file, though it is no cut: a filter that runs out there reports - /// nothing, and decodes as it would read whole. + /// nothing, drops no group the end parts, and undoes a predictor's last row as far as it goes only where it ran out. /// /// Whether the stream reported a filter that stopped at the bound already. /// How many bytes a last group the cut parted would decode to, which the filter did not write. @@ -282,9 +283,13 @@ private static ReadOnlyMemory ApplyOne( // Data a guard of the reader's cut ends early because the reader stopped reading it, or a filter before // this one stopped decoding it, which the guard reported, and data whose end a guard hides ends where the guard // may: that it ran out says nothing of the file. A fault, or a checksum that disagrees, lies in bytes the reader - // did read, and the whole data would meet it at the same point: it is the file's. + // did read, and the whole data would meet it at the same point: it is the file's — but for a checksum that ends + // in white space only a hidden length says is the data's or not, which may be read in place of the checksum's + // last bytes. + var endHidden = (quiet && ending is FlateEnding.TailLost or FlateEnding.ChecksumMissing) || + (silenced && ending == FlateEnding.ChecksumMismatch && PdfCharacters.IsWhitespace(data.Span[^1])); ReportEnding( - quiet && ending is FlateEnding.TailLost or FlateEnding.ChecksumMissing ? FlateEnding.Whole : ending, + endHidden ? FlateEnding.Whole : ending, decoded.Length, faultAt, data.Length, diff --git a/src/AdCodicem.Pdf/Objects/PdfStreamData.cs b/src/AdCodicem.Pdf/Objects/PdfStreamData.cs index 9778f37a..d1323ea6 100644 --- a/src/AdCodicem.Pdf/Objects/PdfStreamData.cs +++ b/src/AdCodicem.Pdf/Objects/PdfStreamData.cs @@ -35,8 +35,8 @@ public abstract class PdfStreamData /// Gets a value indicating whether where the data ends is the guard's to hide, though the data is the file's: an /// endstream proven the stream's own ended it under a /Length a guard hid, with white space before the /// keyword that only the length could say is the data's or not (#224). Decoding then says nothing of the file where the - /// data ends, which it decodes as it would read whole: white space parts no ASCII85 or ASCIIHex group, and falls in a - /// Flate stream's checksum. + /// data ends, which it does not decode as cut: white space parts no ASCII85 or ASCIIHex group, and falls in a Flate + /// stream's checksum. /// internal virtual bool EndHidden => false; diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 2fbbd717..80c6c793 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1112,6 +1112,54 @@ public void Lzw_data_that_lacks_its_end_code_before_a_proven_endstream_past_an_e guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries); } + [Theory] + [InlineData("cut", " ")] + [InlineData("left out", " ")] + [InlineData("cut", "\n\n")] + public void A_checksum_short_of_a_byte_white_space_before_a_proven_endstream_completes_is_no_fault_of_the_file( + string guard, string tail) + { + // Content stream 4's Flate data lacks its checksum's last byte, then white space before its endstream, which the next + // object proves its own. The search took a byte of white space in as data, which only the hidden length says is the + // data's or not, and which the inflater reads as the checksum's last byte: a checksum that disagrees was reported, a + // warning, where read whole the checksum is only short (#224). + byte[] zlib; + + using (var output = new MemoryStream()) + { + using (var stream = new ZLibStream(output, CompressionLevel.Optimal, leaveOpen: true)) + { + stream.Write("BT (Hello) Tj ET"u8); + } + + zlib = output.ToArray()[..^1]; + } + + var data = Encoding.Latin1.GetString(zlib); + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, Page), + [20] = file.Object(20, (guard == "cut" ? new string(' ', 2000) : string.Empty) + zlib.Length.ToString(CultureInfo.InvariantCulture)), + [4] = file.Object(4, $"<< /Length 20 0 R /Filter /FlateDecode >>\nstream\n{data}{tail}endstream"), + [5] = file.Object(5, "(five)"), + }; + file.Table(21, rows, "/Size 21 /Root 1 0 R"); + var limits = guard == "cut" + ? PdfReaderLimits.Default with { MaxObjectLength = 1024 } + : PdfReaderLimits.Default with { MaxIndexEntries = 20 }; + + using var document = PdfDocument.Open(file.Bytes, new PdfReaderOptions { Limits = limits }); + var content = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + content.Decode().ToArray().Should().Equal("BT (Hello) Tj ET"u8.ToArray()); + content.Data.EndHidden.Should().BeTrue(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal( + guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries); + } + private static byte[] ZLibDecompress(byte[] zlib) { using var input = new MemoryStream(zlib); From 85d7c565694f73e6a824a571e52165e6e1aa843a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 20:35:31 +0000 Subject: [PATCH 125/146] fix: find a classic table's xref keyword past however much white space precedes it A cross-reference stream is read past any white space its offset names, as its header is found by reading its object. A classic table was told by the first 32 bytes alone: named 29 or more bytes before its keyword, which they held only in part or not at all, it was not read, the index was rebuilt, and startxref was said to name neither the keyword nor an object. The probe now reads on past white space, comments and a token its edge may cut, as far as a trailer's window reaches. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 18 ++++++++++- .../CrossReferenceRuleTests.cs | 30 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 63a6433a..c43e1226 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3278,10 +3278,24 @@ private XRefSectionState TryReadXRefSection( return section.State; } - using (var probe = _source.GetWindow(absolute, XRefProbeLength)) + // A probe that holds white space and comments alone, or a first token its edge may cut, is read on as far as a + // trailer's window reaches, past which a stream's header is not looked for either: the keyword is the table's + // however far the offset falls short of it. + var probeLength = XRefProbeLength; + + while (true) { + using var probe = _source.GetWindow(absolute, probeLength); var lexer = new PdfLexer(probe.Memory.Span); var first = lexer.Read(); + + if ((first.Kind == PdfTokenKind.EndOfInput || first.End >= probe.Length) && probe.Length == probeLength && + probeLength < _guard.Bound(PdfLimit.Trailer)) + { + probeLength = (int)Math.Min(_guard.Bound(PdfLimit.Trailer), (long)probeLength * 8); + continue; + } + section.Padding = first.Kind == PdfTokenKind.EndOfInput ? probe.Length : first.Start; if (first.IsKeyword("xref"u8)) @@ -3292,6 +3306,8 @@ private XRefSectionState TryReadXRefSection( : XRefSectionState.Malformed; return section.State; } + + break; } section.State = TryReadXRefStream(absolute, section, out previous, candidate); diff --git a/tests/AdCodicem.Pdf.Tests/CrossReferenceRuleTests.cs b/tests/AdCodicem.Pdf.Tests/CrossReferenceRuleTests.cs index d430d982..5023c294 100644 --- a/tests/AdCodicem.Pdf.Tests/CrossReferenceRuleTests.cs +++ b/tests/AdCodicem.Pdf.Tests/CrossReferenceRuleTests.cs @@ -1307,6 +1307,36 @@ public void An_object_a_guard_left_out_of_the_index_is_no_reference_the_file_lac findings.Should().Contain(finding => finding.RuleId == PdfValidationRuleIds.XRefCheckedInPart); } + [Theory] + [InlineData(29)] + [InlineData(32)] + [InlineData(600)] + public void A_startxref_naming_white_space_far_before_a_classic_table_is_imprecise_and_the_table_read(int padding) + { + // Startxref names the start of white space before the xref keyword. The keyword lay past the first 32 bytes the + // reader looks at, or across them: the table was not read, the index was rebuilt, and startxref said to name neither + // the keyword nor an object, where a cross-reference stream is read past however much white space comes first. + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"), + }; + file.Raw(new string(' ', padding)); + var table = file.Table(4, rows, "/Size 4 /Root 1 0 R"); + var bytes = Encoding.Latin1.GetBytes(Encoding.Latin1.GetString(file.Bytes) + .Replace( + string.Create(CultureInfo.InvariantCulture, $"startxref\n{table}\n"), + string.Create(CultureInfo.InvariantCulture, $"startxref\n{table - padding}\n"), + StringComparison.Ordinal)); + using var document = PdfDocument.Open(bytes); + + document.Diagnostics.Should().BeEmpty(); + Single(new PdfValidator().Validate(document), PdfValidationRuleIds.XRefOffsetImprecise).Message.Should().EndWith( + string.Create(CultureInfo.InvariantCulture, $"startxref gives offset {table - padding}, {padding} bytes before the xref keyword.")); + } + [Fact] public void A_startxref_naming_the_line_feed_before_a_cross_reference_stream_is_imprecise() { From e4de97ab0998efd92479879c80d80549bee39888 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 23:15:47 +0000 Subject: [PATCH 126/146] fix: read the rows a predicted row undone completes in data decoded again knowing its end hidden Data decoded again knowing its end hidden held no more rows than the decoding before it, which was meant for a decoding MaxDecodedStreamLength stopped. A decoding no guard stopped, its end taken for the file's, drops a predicted row the data parts, which the second decoding undoes as far as it goes: the rows it completes, before the endstream the rows spell, were left to the guard, and the page whose row it held read as null where the whole read and the direct twin read it. The cap now holds where a guard stopped the decoding before only. Refs #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 9 +++++--- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 18 ++++++++++----- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 22 +++++++++++++++++++ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index c43e1226..d9b82d6f 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -4069,7 +4069,8 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // hidden keeps reached: the whole data holds at least what that decoding read. var reachedBefore = false; - // How many rows the decoding before held, which the data decoded again knowing its end hidden holds no more of. + // How many rows the decoding before held where it stopped at MaxDecodedStreamLength, which the data decoded again + // knowing its end hidden reads no more of. long heldBefore = 0; while (true) @@ -4096,8 +4097,10 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // The data decoded again knowing its end hidden lacks the group a filter dropped there, and a filter after it, // given less, may be given more room than the decoding before gave it, and read rows past where that one stopped - // at a guard on decoding: the whole data holds no more of them than that decoding read (#220). - if (endHidden && held > heldBefore) + // at a guard on decoding: the guard stops it where it stops the data read whole, the decoding before (#220). A + // decoding no guard stopped holds fewer rows only where its end, decoded as the file's, lost a predicted row the + // data decoded again undoes as far as it goes: those it completes are the file's. + if (endHidden && reachedBefore && held > heldBefore) { held = heldBefore; data = data[..(int)(held * row)]; diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index f9a67280..af708a09 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -797,10 +797,12 @@ string Text(long five, long stream) => /// cross-reference stream 7 after 1,015 spaces — a MaxObjectLength of 1,024 parts it after its first digit —, and /// stream 7, Flate in a stored block with a PNG predictor of columns, whose rows are the /// catalog's, two of reserved types that spell endstream after an end-of-line, then the page tree's, the page's, - /// object 5's and its own (/Index [1 1 20 2 2 2 5 1 7 1]). The second is a classic table that places object 5 - /// and names stream 7 through /Prev. The file is valid. + /// object 5's and its own (/Index [1 1 20 2 2 2 5 1 7 1]); with , the free row 0's, + /// the catalog's, the page tree's and the page's come first, then the two that spell endstream, then object 5's + /// and its own (/Index [0 4 20 2 5 1 7 1]). The second is a classic table that places object 5 and names stream 7 + /// through /Prev. The file is valid. /// - public static byte[] PredictedRowsPartedBySpelledEndStream(int columns) + public static byte[] PredictedRowsPartedBySpelledEndStream(int columns, bool pageBefore = false) { using var file = new Writer("1.5"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); @@ -810,9 +812,13 @@ public static byte[] PredictedRowsPartedBySpelledEndStream(int columns) // The data's length does not depend on the offsets its rows hold. byte[] Data(long five, long stream) { - var rows = new List(Encode([(1, file.OffsetOf(1), 0)], Rows.Raw)); + var rows = new List(pageBefore + ? Encode([(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0)], Rows.Raw) + : Encode([(1, file.OffsetOf(1), 0)], Rows.Raw)); rows.AddRange("\nendstream\0\0\0\0"u8.ToArray()); - rows.AddRange(Encode([(1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), (1, five, 0), (1, stream, 0)], Rows.Raw)); + rows.AddRange(pageBefore + ? Encode([(1, five, 0), (1, stream, 0)], Rows.Raw) + : Encode([(1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), (1, five, 0), (1, stream, 0)], Rows.Raw)); while (rows.Count % columns != 0) { @@ -836,7 +842,7 @@ byte[] Data(long five, long stream) 7, string.Create( CultureInfo.InvariantCulture, - $"/Type /XRef /Size 22 /W [1 4 2] /Index [1 1 20 2 2 2 5 1 7 1] /Filter /FlateDecode /DecodeParms << /Predictor 12 /Columns {columns} >> /Root 1 0 R /Length 5 0 R"), + $"/Type /XRef /Size 22 /W [1 4 2] /Index {(pageBefore ? "[0 4 20 2 5 1 7 1]" : "[1 1 20 2 2 2 5 1 7 1]")} /Filter /FlateDecode /DecodeParms << /Predictor 12 /Columns {columns} >> /Root 1 0 R /Length 5 0 R"), Data(file.OffsetOf(5), at)); file.Text("%" + new string('-', 1200) + "\n"); var xref = file.Position; diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 80c6c793..89b5264f 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1717,6 +1717,28 @@ public void A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_le document.GetObject(new PdfObjectId(1)).AsDictionary().Required().IsOfType(PdfName.Catalog).Should().BeTrue(); } + [Fact] + public void Rows_a_predicted_row_undone_as_far_as_it_goes_completes_are_read_though_no_guard_stopped_the_decoding_before() + { + // As above, the rows of objects 0 to 3 before those that spell an endstream, 21 columns: the data up to it decodes to + // a predicted row that holds three, and a second, parted, that holds the page's. Decoded as data whose end the file + // gives, the parted row was dropped; decoded again knowing its end hidden, it is undone as far as it goes, and holds + // the page's row whole. That the data decoded again holds no more rows than the decoding before is for a decoding a + // guard stopped, whose bound a filter after a dropped group may be given more room under (#220). + var file = ChainFiles.PredictedRowsPartedBySpelledEndStream(21, pageBefore: true); + + using (var whole = PdfDocument.Open(file)) + { + whole.Diagnostics.Should().BeEmpty(); + whole.GetObject(new PdfObjectId(3)).AsDictionary().Required().IsOfType(PdfName.Page).Should().BeTrue(); + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + document.GetObject(new PdfObjectId(3)).AsDictionary().Required().IsOfType(PdfName.Page).Should().BeTrue(); + } + [Theory] [InlineData(29)] [InlineData(20)] From c9cf701fc08a5e0f970fdba3395b0fcb8cf1d948 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 23:17:21 +0000 Subject: [PATCH 127/146] fix: take what follows an endstream no endobj follows from the read-on that reached it The proof that an endstream ends a stream's own data takes the first significant byte past the keyword from the read-on that found it, which tells whether an endobj follows, only where one did. Past an endstream the data spells, no endobj after it, it read the white space again from 16 bytes, doubling: thirty streams under a cut /Length, each spelling one before 64 KB of white space, had the searches read close to four times the file, where three are said. They read it about twice. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 5 ++- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 38 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index d9b82d6f..efaeaa2d 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -2038,7 +2038,10 @@ private bool IsStreamSectionAt(long position) end = end < 0 || end >= _source.Length ? -1 : end + StreamEnds.EndObjKeyword.Length; } - var next = end < 0 ? -1 : NextSignificant(end, ProofRead); + // What follows a keyword no endobj follows the reader read on to already, through the white space between. + var next = end < 0 ? -1 : endObj != EndObjState.Present && _endObjStarts.TryGetValue(end, out var after) + ? after + : NextSignificant(end, ProofRead); var follower = (next, next >= 0 && (next == _source.Length || IsKeywordAt(next, "xref"u8) || IsKeywordAt(next, StartXRefKeyword))); _endStreamFollowers[(dataEnd, endObj)] = follower; diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 89b5264f..53d05821 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1577,6 +1577,44 @@ public void The_proofs_of_streams_white_space_follows_read_less_than_three_times document.Reader.EndStreamSearchBytes.Should().BeLessThan(3L * bytes.Length); } + [Fact] + public void The_proofs_of_endstreams_data_spells_before_white_space_read_less_than_three_times_the_file() + { + // Thirty streams whose /Length, object 20, MaxObjectLength cuts spell an endstream in their data, 65,520 bytes of white + // space and a regular byte after it, which no endobj follows. The reader read on through the white space to tell + // whether an endobj follows, and the proof read it again from the keyword to what follows: close to four times the + // file, the most the searches may read, where three were said (#224). + const int Streams = 30; + var run = new string(' ', 65_520); + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, Page), + [20] = file.Object(20, Invariant($"{new string(' ', 1100)}{12 + run.Length}")), + }; + + for (var number = 30; number < 30 + Streams; number++) + { + rows[number] = file.Position; + file.Raw(Invariant($"{number} 0 obj\n<< /Length 20 0 R >>\nstream\nq\nendstream{run}x\nendstream\nendobj\n")); + } + + rows[30 + Streams] = file.Object(30 + Streams, "(end)"); + file.Table(31 + Streams, rows, Invariant($"/Size {31 + Streams} /Root 1 0 R")); + var bytes = file.Bytes; + + using var document = PdfDocument.Open(bytes, ObjectGuard); + + for (var number = 30; number < 30 + Streams; number++) + { + document.GetObject(new PdfObjectId(number)).AsStream().Required().Data.CutByGuard.Should().BeTrue(); + } + + document.Reader.EndStreamSearchBytes.Should().BeLessThan(3L * bytes.Length); + } + [Theory] [InlineData("Rx", false)] [InlineData("RG", false)] From ade55a1bc4e1702bc6e09fd2356b9ead94d6e26d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 23:22:22 +0000 Subject: [PATCH 128/146] test: hold the classic table's probe to a trailer's window and to the end of the file The probe that looks for a classic table's xref keyword reads on past white space as far as MaxTrailerLength reaches. No test named more white space than that, nor white space that runs to the end of the file: a bound reached and read again, a loop, would have passed the suite. The guard is now held reached, reported or thrown, and white space to the end names no table. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../CrossReferenceRuleTests.cs | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/AdCodicem.Pdf.Tests/CrossReferenceRuleTests.cs b/tests/AdCodicem.Pdf.Tests/CrossReferenceRuleTests.cs index 5023c294..5d873091 100644 --- a/tests/AdCodicem.Pdf.Tests/CrossReferenceRuleTests.cs +++ b/tests/AdCodicem.Pdf.Tests/CrossReferenceRuleTests.cs @@ -1337,6 +1337,72 @@ public void A_startxref_naming_white_space_far_before_a_classic_table_is_impreci string.Create(CultureInfo.InvariantCulture, $"startxref gives offset {table - padding}, {padding} bytes before the xref keyword.")); } + [Theory(Timeout = 60_000)] + [InlineData(300, false)] + [InlineData(70_000, false)] + [InlineData(300, true)] + public async Task A_startxref_naming_more_white_space_than_a_trailer_s_window_before_a_classic_table_reaches_the_guard( + int padding, bool throwOnLimit) + { + // The reader reads on past white space for the xref keyword as far as MaxTrailerLength reaches, past which it would + // read the same window again and again: the guard is reached, the index rebuilt, or the guard thrown. + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"), + }; + file.Raw(new string(' ', padding)); + var table = file.Table(4, rows, "/Size 4 /Root 1 0 R"); + var bytes = Encoding.Latin1.GetBytes(Encoding.Latin1.GetString(file.Bytes) + .Replace( + string.Create(CultureInfo.InvariantCulture, $"startxref\n{table}\n"), + string.Create(CultureInfo.InvariantCulture, $"startxref\n{table - padding}\n"), + StringComparison.Ordinal)); + var limits = padding > PdfReaderLimits.Default.MaxTrailerLength + ? PdfReaderLimits.Default + : PdfReaderLimits.Default with { MaxTrailerLength = 256 }; + var options = new PdfReaderOptions { Limits = limits, ThrowOnLimit = throwOnLimit }; + + if (throwOnLimit) + { + await Task.Run(() => FluentActions.Invoking(() => PdfDocument.Open(bytes, options)).Should().Throw(), TestContext.Current.CancellationToken); + return; + } + + using var document = await Task.Run(() => PdfDocument.Open(bytes, options), TestContext.Current.CancellationToken); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitTrailer, PdfDiagnosticCodes.XRefRebuilt); + document.GetObject(new PdfObjectId(3)).AsDictionary().Required().IsOfType(PdfName.Page).Should().BeTrue(); + } + + [Fact] + public void A_startxref_naming_white_space_that_runs_to_the_end_of_the_file_names_no_table() + { + // The white space runs out before the reader's window does: there is nothing more to read on to. + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"), + }; + var table = file.Table(4, rows, "/Size 4 /Root 1 0 R"); + var text = Encoding.Latin1.GetString(file.Bytes).Replace( + string.Create(CultureInfo.InvariantCulture, $"startxref\n{table}\n"), + "startxref\n##########\n", + StringComparison.Ordinal); + var bytes = Encoding.Latin1.GetBytes( + text.Replace("##########", text.Length.ToString("D10", CultureInfo.InvariantCulture), StringComparison.Ordinal) + + new string(' ', 100)); + using var document = PdfDocument.Open(bytes); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.XRefRebuilt); + Single(new PdfValidator().Validate(document), PdfValidationRuleIds.FileStartXRefWrong).Message.Should().EndWith( + "which holds neither the xref keyword nor an object."); + } + [Fact] public void A_startxref_naming_the_line_feed_before_a_cross_reference_stream_is_imprecise() { From 9bfc102d5167f097bd83dee8a603b4c670648ed0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 23:26:51 +0000 Subject: [PATCH 129/146] test: hold where a subsection one of whose elements a guard cut is wrong whatever it holds The subsection one of whose elements a guard cut is the file's fault where the element read is wrong whatever the cut one holds. No test held the edges: a count of 0, valid, a first number of 2147483647, valid with a count of 0 or 1, a first number past it and a count past the object numbers, nor the messages such a fault is said with. Each is now held, and a mutant of each edge or message fails. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 53d05821..898eedf0 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -2802,11 +2802,13 @@ public void A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_n [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [5 0 R -1]", "Index element", null)] [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [5 0 R /x]", "Index element", null)] [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [/x 5 0 R]", "Index element", null)] - [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [5 0 R 3000000000]", "Index element", null, false)] - [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [-1 5 0 R]", "Index element", null, false)] + [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [5 0 R 3000000000]", "Index element", null, false, "its /Index gives a subsection of 3,000,000,000 rows, more than object numbers 0 to 2147483647 hold")] + [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [5 0 R 2147483649]", "Index element", null, false, "its /Index gives a subsection of 2,147,483,649 rows, more than object numbers 0 to 2147483647 hold")] + [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [-1 5 0 R]", "Index element", null, false, "its /Index gives a subsection from object -1, outside object numbers 0 to 2147483647")] + [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [2147483648 5 0 R]", "Index element", null, false, "its /Index gives a subsection from object 2147483648, outside object numbers 0 to 2147483647")] [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [9 0 R -1] /Length 5 0 R", "Length", "cut:0")] public void What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_guard_cut_holds_is_the_file_s( - string entries, string cut, string? nine, bool asReadWhole = true) + string entries, string cut, string? nine, bool asReadWhole = true, string? message = null) { // Stream 7's /Filter, an /Index element, its /W or its /Length names object 5, which MaxObjectLength cuts — the length // where no endstream ends the data. Its /W gives no three widths, or none past 0; its /Index holds no pairs, or a @@ -2851,6 +2853,11 @@ public void What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_gu { guarded.Message.Should().Be(unreadable.Message); } + else + { + guarded.Message.Should().Contain(message); + } + document.WasRepaired.Should().BeTrue(); document.Reader.Structure.Sections.Should().NotContain(section => section.CutByLimit); document.GetObject(new PdfObjectId(1)).AsDictionary().IsOfType(PdfName.Catalog).Should().BeTrue(); @@ -2862,6 +2869,8 @@ public void What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_gu [InlineData("/Type /XRef /Size 8 /W [0 0 5 0 R]", "7", "/W holds the reference 5 0 R")] [InlineData("/Type /XRef /Size 8 /W [1 5 0 R 2] /Index [0 9 0 R]", "4", "/W holds the reference 5 0 R")] [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [5 0 R 8]", "0", "/Index holds the reference 5 0 R")] + [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [5 0 R 0]", "0", "/Index holds the reference 5 0 R")] + [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [2147483647 5 0 R]", "1", "/Index holds the reference 5 0 R")] public void A_section_a_value_s_cut_keeps_from_being_read_is_the_guard_s_where_nothing_read_says_it_is_wrong( string entries, string five, string named) { From 6ad024e21763ed0fc905f6f6b17563b66cf86145 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 23:34:03 +0000 Subject: [PATCH 130/146] fix: charge the file with a section whose data holds too few rows whatever a cut value holds A cross-reference stream a value's cut keeps from being read was the guard's unless its widths or its numbering, as far as read, were wrong. Data whose length the file gives, decoded whole, may hold fewer rows than the counts read declare however few bytes the widths read allow a row: three bytes under a /Size of 8. Read whole, the section lacks rows, the file's, and the index is rebuilt; it was taken for the guard's, the chain ended there and the catalog was lost, or thrown under ThrowOnLimit. Such a section is now the file's to lack rows, there and not looked for nearby, and the index is rebuilt. Data a guard cut or whose end it hides, a /Length the chain cannot read yet, and filters a reference holds say nothing. The test that held the guard's verdict on three bytes holds 56 now. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 147 +++++++++++++++--- .../IO/XRef/XRefSectionRecord.cs | 5 +- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 66 +++++++- 3 files changed, 192 insertions(+), 26 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index efaeaa2d..096ed2b0 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -2906,8 +2906,8 @@ private bool TryReadXRefChain(long startOffset) } // A section a guard cut is where it was named, and is not looked for elsewhere: what lies near it would - // stand for it. - if (section.CutByLimit || !TryRelocateXRefSection(offset, section, out previous, out hybrid)) + // stand for it. So is one that lacks rows whatever a value a guard cut holds, as read whole. + if (section.CutByLimit || section.Incomplete || !TryRelocateXRefSection(offset, section, out previous, out hybrid)) { ReportLostSection(section, naming, offset, namedFrom); break; @@ -2961,7 +2961,7 @@ private bool TryReadXRefChain(long startOffset) _structure.Add(stream); if (TryReadXRefSection(stream.Offset, stream, out _, out _) != XRefSectionState.Read && - (stream.CutByLimit || !TryRelocateXRefSection(hybrid, stream, out _, out _))) + (stream.CutByLimit || stream.Incomplete || !TryRelocateXRefSection(hybrid, stream, out _, out _))) { ReportLostSection(stream, "/XRefStm", hybrid, section.Offset); } @@ -3956,7 +3956,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // value no section read so far places, or what the values read give. var file = UnreadValue(dictionary, out var unreadCut) is { } unplaced && unreadCut is null ? unplaced - : FaultBesideCut(dictionary); + : FaultBesideCut(dictionary, null, out _); section.Fault = file ?? lengthFault; return file is null ? CutByValue(section, past) : XRefSectionState.Malformed; } @@ -3981,9 +3981,12 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect return XRefSectionState.Malformed; } - if (FaultBesideCut(dictionary) is { } file) + // Data whose extent the file gives that cannot hold the rows the values read declare lacks rows whatever the cut + // value holds, as read whole: the section is there, the file's to lack them, and not looked for nearby. + if (FaultBesideCut(dictionary, deferred is null && !xrefStream.Data.CutByGuard && !xrefStream.Data.EndHidden ? xrefStream : null, out var lacking) is { } file) { section.Fault = file; + section.Incomplete = lacking; return XRefSectionState.Malformed; } @@ -4459,17 +4462,28 @@ private XRefSectionState CutByValue(XRefSectionRecord section, PdfObjectId cut) /// /// Says what makes a cross-reference stream unreadable whatever the values a guard cut hold, as a whole read says it: - /// the widths its /W gives, then how it numbers its rows, as far as the values read say. A value a guard cut, or - /// an element of one, says nothing, nor does what depends on it — the /Size an /Index so cut may stand - /// beside —; an /Index that holds no pairs is wrong whatever its elements hold. Null when nothing read is wrong: - /// the section is then the guard's. + /// the widths its /W gives, then how it numbers its rows, as far as the values read say, then, for + /// , whether its data can hold the rows they declare. A value a guard cut, or an element of + /// one, says nothing, nor does what depends on it — the /Size an /Index so cut may stand beside —; an + /// /Index that holds no pairs is wrong whatever its elements hold. Null when nothing read is wrong: the section is + /// then the guard's. /// /// /// followed every value read here, and found none that no section read so far places: a value /// that cannot be read is one a guard cut. /// - private string? FaultBesideCut(PdfDictionary dictionary) + /// The stream's dictionary. + /// The stream, whose data's extent is the file's, or null when it is not. + /// Whether the fault is that the data holds fewer rows than the section declares. + private string? FaultBesideCut(PdfDictionary dictionary, PdfStream? stream, out bool lacking) { + lacking = false; + + // The fewest bytes a row may hold, which the widths read give, and the fewest rows the counts read declare. + long row = 0; + long declared = -1; + var exact = true; + if (Unread(dictionary, PdfName.W, out var w, out _) is null) { if (w is not PdfArray widths || widths.Count < 3) @@ -4496,6 +4510,9 @@ private XRefSectionState CutByValue(XRefSectionRecord section, PdfObjectId cut) } anyWidth |= width > 0; + + // A width is within an int (WidthFault), and three within a long. + row += width.GetValueOrDefault(); } if (known && !anyWidth) @@ -4511,30 +4528,114 @@ private XRefSectionState CutByValue(XRefSectionRecord section, PdfObjectId cut) if (index is null or PdfNull) { - return Unread(dictionary, PdfName.Size, out var size, out _) is null - ? NumberingFault(null, null, size?.AsInteger(), out _) - : null; + if (Unread(dictionary, PdfName.Size, out var size, out _) is not null) + { + return null; + } + + if (NumberingFault(null, null, size?.AsInteger(), out _) is { } sizeFault) + { + return sizeFault; + } + + declared = size?.AsInteger() ?? 0; } + else + { + if (index is not PdfArray ranges || PairsFault(ranges) is not null) + { + return NumberingFault(dictionary.GetRaw(PdfName.Index), index, null, out _); + } + + declared = 0; - if (index is not PdfArray ranges || PairsFault(ranges) is not null) + for (var i = 0; i < ranges.Count; i += 2) + { + var startRead = Follow(PdfName.Index, ranges[i], entry: false, out var start, out _) is null; + var countRead = Follow(PdfName.Index, ranges[i + 1], entry: false, out var count, out _) is null; + + if ((startRead && countRead + ? SubsectionFault(ranges, i, start?.AsInteger(), count?.AsInteger()) + : HalfSubsectionFault(ranges, i, startRead, start?.AsInteger(), countRead, count?.AsInteger())) is { } fault) + { + return fault; + } + + // Each count read is within the object numbers, and an /Index holds no more pairs than a trailer: their sum + // is within a long. A count a guard cut is one more, of 0 rows or more. + if (countRead) + { + declared += count!.AsInteger().GetValueOrDefault(); + } + else + { + exact = false; + } + } + } + + if (stream is null || declared <= 0 || RowsHeldAtMost(stream, Math.Max(row, 1)) is not { } most || declared <= most.Rows) { - return NumberingFault(dictionary.GetRaw(PdfName.Index), index, null, out _); + return null; } - for (var i = 0; i < ranges.Count; i += 2) + lacking = true; + return string.Create( + CultureInfo.InvariantCulture, + $"its data, {most.Length:N0} bytes decoded, holds at most {most.Rows:N0} of the {declared:N0}{(exact ? string.Empty : " or more")} rows its /Index and /Size declare"); + } + + /// + /// Gives how many rows of bytes or more the data of cross-reference stream + /// holds at most, and how long it decodes to, when its filters are written direct and decode + /// it whole; null otherwise: a value of them a reference holds is read as the chain reads one, and its decoding is what + /// the section's reading does. + /// + private static (long Rows, int Length)? RowsHeldAtMost(PdfStream stream, long row) + { + if (!IsWrittenDirect(stream.Dictionary.GetRaw(PdfName.Filter)) || !IsWrittenDirect(stream.Dictionary.GetRaw(PdfName.DecodeParms))) { - var startRead = Follow(PdfName.Index, ranges[i], entry: false, out var start, out _) is null; - var countRead = Follow(PdfName.Index, ranges[i + 1], entry: false, out var count, out _) is null; + return null; + } + + var decoded = Filters.PdfFilterPipeline.Decode(stream, diagnostics: null, endHidden: false, reachedBefore: false, out var leftEncoded, out var cut); + return leftEncoded || cut ? null : (decoded.Length / row, decoded.Length); + } - if ((startRead && countRead - ? SubsectionFault(ranges, i, start?.AsInteger(), count?.AsInteger()) - : HalfSubsectionFault(ranges, i, startRead, start?.AsInteger(), countRead, count?.AsInteger())) is { } fault) + /// + /// Determines whether a stream's /Filter or /DecodeParms is written with no reference: itself, the + /// elements of an array, and the values of a dictionary, its own or an element's. + /// + private static bool IsWrittenDirect(PdfObject? value) + { + if (value is PdfArray elements) + { + for (var i = 0; i < elements.Count; i++) { - return fault; + if (elements[i] is PdfReference || (elements[i] is PdfDictionary parameters && !HoldsNoReference(parameters))) + { + return false; + } } + + return true; } - return null; + return value is not PdfReference && (value is not PdfDictionary dictionary || HoldsNoReference(dictionary)); + } + + /// Determines whether none of the values of is a reference. + private static bool HoldsNoReference(PdfDictionary dictionary) + { + foreach (var entry in dictionary) + { + if (entry.Value is PdfReference) + { + return false; + } + } + + return true; } /// diff --git a/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs b/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs index 8d9911b3..07d5911c 100644 --- a/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs +++ b/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs @@ -64,7 +64,10 @@ public XRefSectionRecord(string namedBy, long namedOffset, long namedFrom) /// public string? RefusedRow { get; set; } - /// Gets a value indicating whether a cross-reference stream holds fewer rows than it declares. + /// + /// Gets a value indicating whether a cross-reference stream holds fewer rows than it declares: read, the rows it holds + /// placing objects, or left unread beside a value a guard cut, whose data holds too few whatever the value holds. + /// public bool Incomplete { get; set; } /// diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 898eedf0..9b5debc2 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -2804,6 +2804,7 @@ public void A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_n [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [/x 5 0 R]", "Index element", null)] [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [5 0 R 3000000000]", "Index element", null, false, "its /Index gives a subsection of 3,000,000,000 rows, more than object numbers 0 to 2147483647 hold")] [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [5 0 R 2147483649]", "Index element", null, false, "its /Index gives a subsection of 2,147,483,649 rows, more than object numbers 0 to 2147483647 hold")] + [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [5 0 R 2147483648]", "Index element", null, false, "its data, 56 bytes decoded, holds at most 8 of the 2,147,483,648 rows its /Index and /Size declare")] [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [-1 5 0 R]", "Index element", null, false, "its /Index gives a subsection from object -1, outside object numbers 0 to 2147483647")] [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [2147483648 5 0 R]", "Index element", null, false, "its /Index gives a subsection from object 2147483648, outside object numbers 0 to 2147483647")] [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [9 0 R -1] /Length 5 0 R", "Length", "cut:0")] @@ -2927,9 +2928,9 @@ public void A_place_a_section_may_have_moved_to_that_a_value_past_a_guard_s_cut_ } [Theory] - [InlineData("<< /Type /Font /W 5 0 R /Size 8 /Length 3 >>\nstream\nabc\nendstream", "[1 4 2]")] + [InlineData("<< /Type /Font /W 5 0 R /Size 8 /Length 56 >>\nstream\n{data}\nendstream", "[1 4 2]")] [InlineData("<< /Type /Font /W [1 4 2] /Size 8 /Length 5 0 R >>\nstream\nabc\nendstreaX", "3")] - [InlineData("<< /Type /Xref /W [1 5 0 R 2] /Size 8 /Length 3 >>\nstream\nabc\nendstream", "4")] + [InlineData("<< /Type /Xref /W [1 5 0 R 2] /Size 8 /Length 56 >>\nstream\n{data}\nendstream", "4")] public void A_stream_whose_value_a_guard_cut_where_the_chain_names_a_section_is_the_section_the_guard_cut_whatever_its_type( string six, string five) { @@ -2938,6 +2939,7 @@ public void A_stream_whose_value_a_guard_cut_where_the_chain_names_a_section_is_ // the chain names it, a stream with a /W is read as the section whole, whatever its type: the value hides whether its // rows are, and the section is the guard's. It was taken for no section, the section looked for nearby, and a stream // found there — an older revision's — read in its place (#263). + six = six.Replace("{data}", new string('0', 56), StringComparison.Ordinal); var built = ChainFiles.UnderAnUpdate( "/Type /XRef /Size 8 /W [1 4 2] /Length {length}", five == "3" ? new string(' ', 1100) + five : five + new string(' ', 1100), @@ -2960,6 +2962,66 @@ public void A_stream_whose_value_a_guard_cut_where_the_chain_names_a_section_is_ document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); } + [Theory] + [InlineData("<< /Type /Font /W 5 0 R /Size 8 /Length 3 >>\nstream\nabc\nendstream", "[1 4 2]", "holds at most 3 of the 8 rows")] + [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Length 3 >>\nstream\nabc\nendstream", "[1 4 2]", "holds at most 3 of the 8 rows")] + [InlineData("<< /Type /Xref /W [1 5 0 R 2] /Size 8 /Length 3 >>\nstream\nabc\nendstream", "4", "holds at most 1 of the 8 rows")] + [InlineData("<< /Type /XRef /W [1 4 2] /Index [5 0 R 8] /Length 3 >>\nstream\nabc\nendstream", "0", "holds at most 0 of the 8 rows")] + [InlineData("<< /Type /XRef /W [1 4 2] /Index [5 0 R 1 9 5 0 R] /Length 3 >>\nstream\nabc\nendstream", "0", "holds at most 0 of the 1 or more rows")] + [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Filter /FlateDecode /Length {length} >>\nstream\n{flate}\nendstream", "[1 4 2]", "its data, 5 bytes decoded, holds at most 5 of the 8 rows")] + public void A_section_whose_data_holds_fewer_rows_than_it_declares_whatever_a_value_a_guard_cut_holds_is_the_file_s_to_lack( + string six, string five, string fault) + { + // As above, object 6's data, whose length it gives, holds fewer rows than its /Size or the /Index counts read declare, + // however few bytes the widths read and the one cut give a row: read whole, it lacks rows, the file's, and the index + // is rebuilt. It was taken for the guard's: the chain ended, nothing was rebuilt and the catalog was lost (#263). It + // is not looked for nearby either, where an older revision's stream would stand for it. + using var output = new MemoryStream(); + + using (var zlib = new ZLibStream(output, CompressionLevel.NoCompression, leaveOpen: true)) + { + zlib.Write(new byte[5]); + } + + var flate = output.ToArray(); + six = six + .Replace("{flate}", Encoding.Latin1.GetString(flate), StringComparison.Ordinal) + .Replace("{length}", Invariant($"{flate.Length}"), StringComparison.Ordinal); + var built = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] /Length {length}", + five + new string(' ', 1100), + compressed: false, + predicted: false, + placed: true, + six: six, + placedToo: new Dictionary { [8] = "(eight)" }); + var text = Encoding.Latin1.GetString(built); + var stream = text.IndexOf("\n7 0 obj", StringComparison.Ordinal) + 1; + var sixAt = text.IndexOf("\n6 0 obj", StringComparison.Ordinal) + 1; + var file = Encoding.Latin1.GetBytes(text.Replace(Invariant($"/Prev {stream} "), Invariant($"/Prev {sixAt} "), StringComparison.Ordinal)); + + using (var whole = PdfDocument.Open(file)) + { + whole.WasRepaired.Should().BeTrue(); + whole.Reader.Structure.Sections[^1].Incomplete.Should().BeTrue(); + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + var section = document.Reader.Structure.Sections[^1]; + section.Offset.Should().Be(sixAt); + section.State.Should().Be(XRefSectionState.Malformed); + section.CutByLimit.Should().BeFalse(); + section.Incomplete.Should().BeTrue(); + section.Fault.Should().Contain(fault); + document.WasRepaired.Should().BeTrue(); + document.GetObject(new PdfObjectId(1)).AsDictionary().Required().IsOfType(PdfName.Catalog).Should().BeTrue(); + + // The rebuild reaches the object the guard cut, as it reaches every object. + document.Diagnostics.Select(entry => entry.Code).Should().Equal( + PdfDiagnosticCodes.XRefSectionUnreadable, PdfDiagnosticCodes.XRefRebuilt, PdfDiagnosticCodes.LimitObject); + } + [Fact] public void A_cross_reference_stream_that_misspells_its_type_and_whose_value_a_guard_cut_is_the_section_the_guard_cut() { From 17434fe6c2ae31592e14c562eb22264dfee528d0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 23:48:28 +0000 Subject: [PATCH 131/146] fix: leave to the guard only the white space and comments a hidden length may hold Data an endstream proven its own ends under a /Length a guard hid decoded to its end, every filter's end report silenced, a Flate checksum silenced where the data ended in white space, and a predicted row undone wherever a filter of the chain ran out. That silenced a wrong checksum whose bytes all lie before the white space, reported one a comment completes, decoded white space and a comment's digits as LZW codes or hex, faults the whole read never meets, undid rows behind ASCIIHex that ended where its format says, and charged the file with an object stream's last member the end-of-line the search left out had shortened. The first filter now decodes the data as taken, and where it ends it as its format says with nothing to report, that is the data read whole. Otherwise it decodes it again up to where the run of white space and comments that ends it starts: what it meets before is the file's, running out there says nothing, and where its output may lack what the run holds, Flate's tail or LZW and RunLength codes, the data's end is the guard's, as cut, for the filters after it, the predictor and an object stream's reader. The filters after a first that ended whole decode as read whole. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../IO/Filters/PdfFilterPipeline.cs | 176 +++++++++++--- src/AdCodicem.Pdf/Objects/PdfStreamData.cs | 8 +- .../AdCodicem.Pdf.Tests/FilterDamageTests.cs | 16 +- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 224 ++++++++++++++++++ 4 files changed, 385 insertions(+), 39 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs index 0f296562..16ab4b9d 100644 --- a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs +++ b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs @@ -40,8 +40,10 @@ public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diag /// the reader lets data pass through, or a value the filters read that a guard cut or left out of the index: what comes /// back is then the data still encoded by the filters past it, no part of what the stream decodes to —, and whether one /// cut the data, as a reader of its rows or its objects must know. Data a proven endstream ended past white space - /// that only a hidden length says is the data's or not () is not decoded as cut, - /// and a filter that runs out at its end, or whose checksum ends in that white space, says nothing of the file. + /// and comments that only a hidden length says are the data's or not () decodes as + /// read whole where its first filter ends it as its format says; otherwise up to where that run starts, a filter that + /// runs out there saying nothing of the file, and the data cut by a guard where the first filter's output may lack what + /// the run holds. /// /// The stream to decode. /// Receives what decoding met; when none are supplied, the stream's document does. @@ -51,8 +53,9 @@ public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diag /// /// /// Whether the data decoded is cut by a guard: the stream's own data, which the reader read in part, or what a filter - /// stopped at . A guard that loading a value the filters - /// read reaches is that value's, and does not cut the data. + /// stopped at , or what a first filter that ran out where + /// a guard hides the data's end decoded to. A guard that loading a value the filters read reaches is that value's, and + /// does not cut the data. /// /// /// Decoding reaches one of its document's guards, and the document was opened to throw when it does. @@ -160,11 +163,15 @@ private static ReadOnlyMemory DecodeChain( // and against what the filters before the last share. var cut = stream.Data.CutByGuard || endHidden; - // Data whose endstream is proven its own past white space only a hidden length says is the data's or not ends where - // the file's does but for that white space, which parts no ASCII85 or ASCIIHex group and falls in a Flate stream's - // checksum: no group is dropped and no row undone for it, so that it decodes as read whole, but for LZW or RunLength - // data that lacks its end marker and takes the white space in; a filter that runs out there says nothing of the file. - var silenced = stream.Data.EndHidden; + // Data whose endstream is proven its own past white space and comments only a hidden length says are the data's or + // not (#224): the first filter decodes it as the search took it, and where it ends it there as its format says, + // with nothing to report, that is the data read whole. Otherwise it decodes it again up to where that run starts: + // what it meets before is the file's, its running out there says nothing of the file, and where its output may + // lack what the run holds — Flate that lost its tail, LZW or RunLength data without its end marker —, the output's + // end is the guard's, as of data a guard cut, for the filters after it and the reader of what the stream decodes + // to. White space and comments decode to nothing in ASCII85 or ASCIIHex data, whose output is whole. + var hidden = stream.Data.EndHidden && !cut; + var endCut = false; // A guard a decoding of the same data reached before was reported then: its verdict stands, whatever this one meets. var limitReported = reachedBefore; @@ -187,6 +194,31 @@ private static ReadOnlyMemory DecodeChain( var last = step.Index == chain.LastIndex; var maxLength = last ? bound : (int)Math.Clamp(shared, 0, bound); + var given = data; + var silenced = false; + + if (hidden) + { + hidden = false; + + if (TryDecodeWhole(step, data, maxLength, out var whole)) + { + data = ApplyPredictor(whole, step, diagnostics, stream.Data.Position, cut: false); + shared -= data.Length; + continue; + } + + silenced = true; + data = data[..HiddenRunStart(data.Span)]; + + // Data that is all white space and comments decodes to nothing, which may lack all the run holds. + if (data.IsEmpty) + { + endCut = true; + break; + } + } + data = ApplyOne( step, data, @@ -198,14 +230,21 @@ private static ReadOnlyMemory DecodeChain( silenced, ref limitReported, out var dropped, - out var failed); + out var failed, + out var ranOut); // Data a filter could not decode at all comes back as the filters before it left it, as it does at a filter the // library does not know: the filters after it would read what it encoded, and each fail on it again. if (failed) { - cutByGuard = stream.Data.CutByGuard || limitReported; - return data; + cutByGuard = stream.Data.CutByGuard || limitReported || endCut; + return given; + } + + if (silenced && ranOut) + { + cut = true; + endCut = true; } // A group the filter dropped at the cut is data the whole would decode to, and leaves the filters after it no @@ -218,10 +257,86 @@ private static ReadOnlyMemory DecodeChain( // A value a guard cut or left out of the index was reported where the guard was reached: the chain ends there in // silence. leftEncoded = chain.End is FilterChainEnd.Limit or FilterChainEnd.Guard; - cutByGuard = stream.Data.CutByGuard || limitReported; + cutByGuard = stream.Data.CutByGuard || limitReported || endCut; return data; } + /// + /// Gives where the run of white space and comments that ends starts: the bytes only a hidden + /// length says are the data's or not, as the gap between a length and its endstream may hold them (#174), a + /// comment there starting past white space, at the first % its line holds so. + /// + /// Each line is read twice at most: the run is found in time linear in the data. + private static int HiddenRunStart(ReadOnlySpan data) + { + var start = data.Length; + + while (true) + { + while (start > 0 && PdfCharacters.IsWhitespace(data[start - 1])) + { + start--; + } + + // A comment runs to the end of its line, which the white space passed ends, or the data's end does: it starts at + // the first % its line holds past white space, or at the line's start, past the end-of-line before it. + var line = start; + + while (line > 0 && data[line - 1] is not ((byte)'\r' or (byte)'\n')) + { + line--; + } + + var comment = -1; + + for (var i = line; i < start && comment < 0; i++) + { + if (data[i] == (byte)'%' && (i == line || PdfCharacters.IsWhitespace(data[i - 1]))) + { + comment = i; + } + } + + if (comment < 0) + { + return start; + } + + start = comment; + } + } + + /// + /// Decodes data whose end a guard hides as the search took it, and determines whether the filter ended it as its format + /// says, with nothing to report: its end marker or checksum met, no fault, the bound not reached. + /// + private static bool TryDecodeWhole(in FilterStep step, ReadOnlyMemory data, int maxLength, out byte[] decoded) + { + var name = step.Name; + bool limited; + + if (name == PdfName.FlateDecode) + { + return FlateFilter.TryDecode(data, out decoded, out var repaired, out var ending, out _, out limited, maxLength) && + !repaired && ending == FlateEnding.Whole && !limited; + } + + if (name == PdfName.LZWDecode) + { + decoded = LzwFilter.Decode(data.Span, step.Parameters.EarlyChange, out limited, out var undefinedCode, out var lzwEnding, maxLength); + return !limited && undefinedCode < 0 && lzwEnding == DataEnding.Marked; + } + + DataFaults faults; + decoded = name == PdfName.ASCII85Decode + ? Ascii85Filter.Decode(data.Span, out limited, out faults, maxLength, cut: false) + : name == PdfName.ASCIIHexDecode + ? AsciiHexFilter.Decode(data.Span, out limited, out faults, maxLength, cut: false) + : RunLengthFilter.Decode(data.Span, out limited, out faults, maxLength); + return !limited && faults.Ending == DataEnding.Marked && faults.Passed == 0 && faults.Overflowing == 0 && + faults.BrokenMarker == 0 && !faults.UnfinishedMarker && !faults.Introducer && !faults.LoneDigit; + } + /// /// Determines whether a filter produces an image format that should be left encoded. /// @@ -240,12 +355,18 @@ public static bool IsImageFilter(PdfName name) => /// Whether is data the reader cut; set when this filter stops at the bound too. /// /// - /// Whether where the data ends says nothing of the file, though it is no cut: a filter that runs out there reports + /// Whether where the data ends says nothing of the file, though it is no cut — data whose end a guard hides, up to the + /// white space and comments only the hidden length says are the data's or not —: a filter that runs out there reports /// nothing, drops no group the end parts, and undoes a predictor's last row as far as it goes only where it ran out. /// /// Whether the stream reported a filter that stopped at the bound already. /// How many bytes a last group the cut parted would decode to, which the filter did not write. /// Whether the filter could decode nothing, which ends the chain. + /// + /// Whether the filter ran out at the end of its data where its output may lack what more data would decode to: Flate + /// that lost its tail, LZW or RunLength data without its end marker. White space and comments, which decode to nothing in + /// ASCII85 and ASCIIHex data, are no more data for them. + /// private static ReadOnlyMemory ApplyOne( in FilterStep step, ReadOnlyMemory data, @@ -257,7 +378,8 @@ private static ReadOnlyMemory ApplyOne( bool silenced, ref bool limitReported, out int dropped, - out bool failed) + out bool failed, + out bool ranOut) { var name = step.Name; var maxLength = bounds.MaxLength; @@ -265,6 +387,7 @@ private static ReadOnlyMemory ApplyOne( var quiet = given || silenced; dropped = 0; failed = false; + ranOut = false; if (name == PdfName.FlateDecode) { @@ -283,23 +406,15 @@ private static ReadOnlyMemory ApplyOne( // Data a guard of the reader's cut ends early because the reader stopped reading it, or a filter before // this one stopped decoding it, which the guard reported, and data whose end a guard hides ends where the guard // may: that it ran out says nothing of the file. A fault, or a checksum that disagrees, lies in bytes the reader - // did read, and the whole data would meet it at the same point: it is the file's — but for a checksum that ends - // in white space only a hidden length says is the data's or not, which may be read in place of the checksum's - // last bytes. - var endHidden = (quiet && ending is FlateEnding.TailLost or FlateEnding.ChecksumMissing) || - (silenced && ending == FlateEnding.ChecksumMismatch && PdfCharacters.IsWhitespace(data.Span[^1])); - ReportEnding( - endHidden ? FlateEnding.Whole : ending, - decoded.Length, - faultAt, - data.Length, - diagnostics, - position); + // did read, and the whole data would meet it at the same point: it is the file's. + var endHidden = quiet && ending is FlateEnding.TailLost or FlateEnding.ChecksumMissing; + ReportEnding(endHidden ? FlateEnding.Whole : ending, decoded.Length, faultAt, data.Length, diagnostics, position); Reached(limited, name, bounds.Shared, diagnostics, position, guard, ref cut, ref limitReported); // Data that ran out where a guard hides its end may have lost the end of its last row there: it is undone as far // as it goes, as of data a guard cut. - return ApplyPredictor(decoded, step, diagnostics, position, cut || (silenced && ending == FlateEnding.TailLost)); + ranOut = ending == FlateEnding.TailLost; + return ApplyPredictor(decoded, step, diagnostics, position, cut || (silenced && ranOut)); } if (name == PdfName.LZWDecode) @@ -334,8 +449,8 @@ private static ReadOnlyMemory ApplyOne( } Reached(limited, name, bounds.Shared, diagnostics, position, guard, ref cut, ref limitReported); - return ApplyPredictor( - decoded, step, diagnostics, position, cut || (silenced && ending is DataEnding.Unmarked or DataEnding.UnmarkedInsideCode)); + ranOut = undefinedCode < 0 && ending is DataEnding.Unmarked or DataEnding.UnmarkedInsideCode; + return ApplyPredictor(decoded, step, diagnostics, position, cut || (silenced && ranOut)); } byte[] text; @@ -361,6 +476,7 @@ private static ReadOnlyMemory ApplyOne( ReportFaults(faults, name, data.Length, textLimited, ended: !quiet && !textLimited, diagnostics, position); Reached(textLimited, name, bounds.Shared, diagnostics, position, guard, ref cut, ref limitReported); dropped = faults.Dropped; + ranOut = name == PdfName.RunLengthDecode && faults.Ending != DataEnding.Marked; return text; } diff --git a/src/AdCodicem.Pdf/Objects/PdfStreamData.cs b/src/AdCodicem.Pdf/Objects/PdfStreamData.cs index d1323ea6..85e36463 100644 --- a/src/AdCodicem.Pdf/Objects/PdfStreamData.cs +++ b/src/AdCodicem.Pdf/Objects/PdfStreamData.cs @@ -33,10 +33,10 @@ public abstract class PdfStreamData /// /// Gets a value indicating whether where the data ends is the guard's to hide, though the data is the file's: an - /// endstream proven the stream's own ended it under a /Length a guard hid, with white space before the - /// keyword that only the length could say is the data's or not (#224). Decoding then says nothing of the file where the - /// data ends, which it does not decode as cut: white space parts no ASCII85 or ASCIIHex group, and falls in a Flate - /// stream's checksum. + /// endstream proven the stream's own ended it under a /Length a guard hid, with white space, or comments, + /// before the keyword that only the length could say are the data's or not (#224). The data decodes as read whole where + /// its first filter ends it as its format says; otherwise up to where that run starts, what lies before it the file's, + /// a filter that runs out there saying nothing of the file, and the end of what it may lack the guard's. /// internal virtual bool EndHidden => false; diff --git a/tests/AdCodicem.Pdf.Tests/FilterDamageTests.cs b/tests/AdCodicem.Pdf.Tests/FilterDamageTests.cs index e8df000b..06429a04 100644 --- a/tests/AdCodicem.Pdf.Tests/FilterDamageTests.cs +++ b/tests/AdCodicem.Pdf.Tests/FilterDamageTests.cs @@ -1938,10 +1938,11 @@ private static byte[] CompressRaw(byte[] data) } /// - /// Encodes as the PDF variant of LZW does: a clear first, the end-of-data code last, and the code width - /// growing one code early when is 1. + /// Encodes as the PDF variant of LZW does: a clear first, the end-of-data code last unless + /// says not to, and the code width growing one code early when + /// is 1. /// - private static byte[] LzwEncode(byte[] data, int earlyChange = 1) + internal static byte[] LzwEncode(byte[] data, int earlyChange = 1, bool endCode = true) { var table = new Dictionary(StringComparer.Ordinal); for (var i = 0; i < 256; i++) @@ -1976,12 +1977,17 @@ private static byte[] LzwEncode(byte[] data, int earlyChange = 1) } codes.Add((table[current], bits)); - codes.Add((257, bits)); + + if (endCode) + { + codes.Add((257, bits)); + } + return Pack(codes); } /// Packs LZW codes nine bits each, most significant bit first, as the filter reads them. - private static byte[] NineBitCodes(params int[] codes) => Pack([.. codes.Select(code => (code, 9))]); + internal static byte[] NineBitCodes(params int[] codes) => Pack([.. codes.Select(code => (code, 9))]); private static byte[] Pack(List<(int Code, int Bits)> codes) { diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 9b5debc2..96008177 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1116,6 +1116,8 @@ public void Lzw_data_that_lacks_its_end_code_before_a_proven_endstream_past_an_e [InlineData("cut", " ")] [InlineData("left out", " ")] [InlineData("cut", "\n\n")] + [InlineData("cut", "\r\n%c\r\n")] + [InlineData("left out", " %c\n")] public void A_checksum_short_of_a_byte_white_space_before_a_proven_endstream_completes_is_no_fault_of_the_file( string guard, string tail) { @@ -1160,6 +1162,228 @@ public void A_checksum_short_of_a_byte_white_space_before_a_proven_endstream_com guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries); } + [Theory] + [InlineData("cut", "\n")] + [InlineData("cut", "\n\n")] + [InlineData("left out", " \n")] + [InlineData("cut", "\r\n\r\n")] + public void A_checksum_that_disagrees_before_the_white_space_a_hidden_length_leaves_is_the_file_s(string guard, string tail) + { + // Content stream 4's Flate data is whole, its checksum's first byte wrong, then white space before its endstream, + // which the next object proves its own. Its last byte is no white space, nor completed by any: the checksum disagrees + // whatever the hidden length holds, as read whole. It said nothing where the data taken ended in white space (#224). + var zlib = Zlib("BT (Hello) Tj ET"u8); + zlib[^4] ^= 0x55; + var file = HiddenLengthFile(guard, "/Filter /FlateDecode", zlib, tail); + + using (var whole = PdfDocument.Open(file)) + { + whole.GetObject(new PdfObjectId(4)).AsStream().Required().Decode().ToArray().Should().Equal("BT (Hello) Tj ET"u8.ToArray()); + whole.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.FilterChecksumMismatch); + } + + using var document = PdfDocument.Open(file, HiddenLengthOptions(guard)); + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + stream.Decode().ToArray().Should().Equal("BT (Hello) Tj ET"u8.ToArray()); + stream.Data.EndHidden.Should().BeTrue(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(GuardCode(guard), PdfDiagnosticCodes.FilterChecksumMismatch); + } + + [Theory] + [InlineData("cut", 12, 60)] + [InlineData("left out", 12, 60)] + [InlineData("cut", 2, 60)] + [InlineData("cut", 12, 30)] + public void A_filter_after_one_that_ends_where_its_format_says_decodes_as_read_whole_whatever_hides_the_end( + string guard, int predictor, int kept) + { + // Content stream 4's ASCIIHex data, its > end marker in place before the end-of-line and the endstream the next object + // proves its own, decodes to Flate data that stores two predicted rows of 41 bytes, cut short after so many. The + // ASCIIHex data ends where its format says, before the white space: the Flate data it decodes to is whole, and lost + // its tail as read whole, the file's. It was decoded as data the hidden end ran short, its parted row undone, or a + // predictor's warning left unsaid (#224). + var rows = new byte[82]; + + for (var i = 0; i < rows.Length; i++) + { + rows[i] = i % 41 == 0 ? (byte)0 : (byte)('a' + (i % 26)); + } + + byte[] stored = [0x78, 0x01, 0x01, 82, 0, unchecked((byte)~82), 0xFF, .. rows]; + var hex = Convert.ToHexString(stored[..(7 + kept)]) + ">"; + var file = HiddenLengthFile( + guard, + Invariant($"/Filter [/ASCIIHexDecode /FlateDecode] /DecodeParms [null << /Predictor {predictor} /Columns 40 >>]"), + Encoding.ASCII.GetBytes(hex), + "\n"); + byte[] whole; + string[] faults; + + using (var document = PdfDocument.Open(file)) + { + whole = document.GetObject(new PdfObjectId(4)).AsStream().Required().Decode().ToArray(); + faults = [.. document.Diagnostics.Select(entry => entry.Message)]; + faults.Should().NotBeEmpty(); + } + + using var guarded = PdfDocument.Open(file, HiddenLengthOptions(guard)); + var stream = guarded.GetObject(new PdfObjectId(4)).AsStream().Required(); + + stream.Decode().ToArray().Should().Equal(whole); + stream.Data.EndHidden.Should().BeTrue(); + guarded.Diagnostics.Skip(1).Select(entry => entry.Message).Should().Equal(faults); + } + + [Theory] + [InlineData("cut", 2, 5)] + [InlineData("left out", 2, 5)] + [InlineData("cut", 12, 4)] + [InlineData("left out", 12, 14)] + public void Lzw_rows_a_hidden_end_leaves_a_byte_short_are_undone_as_far_as_they_go(string guard, int predictor, int columns) + { + // Content stream 4's LZW data, a clear code and fifteen literal codes, lacks its end-of-data code: its last byte, + // 0x0D, ends the last code, then an end-of-line and the endstream the next object proves its own. The search leaves + // the carriage return out with the line feed, which only the hidden length says is the data's or not: the codes run + // out a byte short, which says nothing of the file, and the row it parts is the guard's, undone as far as it goes. + // It was dropped, left predicted, or, the data shorter than a row of 14 columns, reported as rows longer than the + // data (#224). + int[] plain = predictor == 2 + ? [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 20, 30, 13] + : [0, 1, 2, 3, 4, 2, 1, 1, 1, 1, 2, 3, 5, 7, 13]; + var raw = FilterDamageTests.NineBitCodes([256, .. plain]); + raw[^1].Should().Be(0x0D); + var file = HiddenLengthFile( + guard, Invariant($"/Filter /LZWDecode /DecodeParms << /Predictor {predictor} /Columns {columns} >>"), raw, "\n"); + byte[] whole; + + using (var document = PdfDocument.Open(file)) + { + whole = document.GetObject(new PdfObjectId(4)).AsStream().Required().Decode().ToArray(); + } + + using var guarded = PdfDocument.Open(file, HiddenLengthOptions(guard)); + var stream = guarded.GetObject(new PdfObjectId(4)).AsStream().Required(); + + stream.Decode().ToArray().Should().Equal(whole[..^1]); + stream.Data.EndHidden.Should().BeTrue(); + guarded.Diagnostics.Select(entry => entry.Code).Should().Equal(GuardCode(guard)); + } + + [Theory] + [InlineData("cut", "/LZWDecode", "\n\n\n\n")] + [InlineData("left out", "/LZWDecode", "\t\t\t\t\t\n")] + [InlineData("cut", "/ASCIIHexDecode", "\n%c\n")] + [InlineData("left out", "/ASCIIHexDecode", " %c\n")] + public void White_space_and_comments_a_hidden_length_leaves_are_not_decoded_as_data_that_lacks_its_end_marker( + string guard, string filter, string tail) + { + // Content stream 4's data, LZW or ASCIIHex, lacks its end marker, then white space, or a comment, before the + // endstream the next object proves its own: what it decodes to, read whole, is "q", and a missing end marker. The + // LZW data read the white space as more codes, one of them undefined, and the ASCIIHex data a comment's digit, its % + // a byte passed over: faults the file's length leaves out, and bytes it does not decode to (#224). + var data = filter == "/LZWDecode" ? [0x80, 0x1C, 0x40] : "71"u8.ToArray(); + var file = HiddenLengthFile(guard, "/Filter " + filter, data, tail); + + using (var whole = PdfDocument.Open(file)) + { + whole.GetObject(new PdfObjectId(4)).AsStream().Required().Decode().ToArray().Should().Equal("q"u8.ToArray()); + whole.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.FilterFailed); + } + + using var document = PdfDocument.Open(file, HiddenLengthOptions(guard)); + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + stream.Decode().ToArray().Should().Equal("q"u8.ToArray()); + stream.Data.EndHidden.Should().BeTrue(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(GuardCode(guard)); + } + + [Theory] + [InlineData("cut")] + [InlineData("left out")] + public void An_object_stream_whose_last_code_a_hidden_end_leaves_a_byte_short_ends_where_the_guard_hides(string guard) + { + // Object stream 6's LZW data lacks its end-of-data code, and its last byte, 0x0A, which ends the last code, the + // catalog's closing " >>", stands right before the endstream the next object proves its own; its /Length, object 20, + // is cut or left out of the index. The search took it for + // the end-of-line before the keyword, which only the hidden length says is the data's or not: the codes run out a + // byte short, which says nothing of the file. The catalog's dictionary, left unclosed, was reported as the file's + // syntax fault (#224). + var letters = "ggjlxbmorogsxwbojsdnaqehqoilkdhznwbupwnsouprkjamamuxnotnanksexotmnfxewtjihojygfiwdqvarkhjijdmbaavguqtbvkp"; + var first = $"<< /A ({letters}) /B << /C 1 >> /D << /E 2 >> >>"; + var plain = Invariant($"9 0 1 {first.Length + 1} {first} << /Type /Catalog /Pages 2 0 R >>"); + var lzw = FilterDamageTests.LzwEncode(Encoding.ASCII.GetBytes(plain), endCode: false); + lzw[^1].Should().Be(0x0A); + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = (2, 6, 1), + [2] = (1, file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), 0), + [3] = (1, file.Object(3, Page), 0), + [20] = (1, file.Object(20, (guard == "cut" ? new string(' ', 1100) : string.Empty) + Invariant($"{lzw.Length}")), 0), + [6] = (1, file.Position, 0), + [9] = (2, 6, 0), + }; + file.Raw(Invariant($"6 0 obj\n<< /Type /ObjStm /N 2 /First {plain.IndexOf('<', StringComparison.Ordinal)} /Filter /LZWDecode /Length 20 0 R >>\nstream\n")); + file.Raw(Encoding.Latin1.GetString(lzw) + "endstream\nendobj\n"); + file.CrossReferenceStream(7, 21, rows, "/Root 1 0 R"); + + using (var whole = PdfDocument.Open(file.Bytes)) + { + whole.GetObject(new PdfObjectId(1)).AsDictionary().Required().IsOfType(PdfName.Catalog).Should().BeTrue(); + whole.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.FilterFailed); + } + + using var document = PdfDocument.Open(file.Bytes, HiddenLengthOptions(guard)); + + document.GetObject(new PdfObjectId(1)).AsDictionary().Required().IsOfType(PdfName.Catalog).Should().BeTrue(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(GuardCode(guard)); + } + + /// + /// A file whose content stream 4 holds and before its endstream, which + /// object 5 after it proves its own, its /Length object 20, which a MaxObjectLength of 1,024 cuts, or, left + /// out, a MaxIndexEntries of 20 leaves out of the index. + /// + private static byte[] HiddenLengthFile(string guard, string entries, byte[] data, string tail) + { + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, Page), + [20] = file.Object(20, (guard == "cut" ? new string(' ', 2000) : string.Empty) + data.Length.ToString(CultureInfo.InvariantCulture)), + [4] = file.Object(4, $"<< /Length 20 0 R {entries} >>\nstream\n{Encoding.Latin1.GetString(data)}{tail}endstream"), + [5] = file.Object(5, "(five)"), + }; + file.Table(21, rows, "/Size 21 /Root 1 0 R"); + return file.Bytes; + } + + private static PdfReaderOptions HiddenLengthOptions(string guard) => new() + { + Limits = guard == "cut" + ? PdfReaderLimits.Default with { MaxObjectLength = 1024 } + : PdfReaderLimits.Default with { MaxIndexEntries = 20 }, + }; + + private static string GuardCode(string guard) => + guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries; + + private static byte[] Zlib(ReadOnlySpan data) + { + using var output = new MemoryStream(); + + using (var stream = new ZLibStream(output, CompressionLevel.Optimal, leaveOpen: true)) + { + stream.Write(data); + } + + return output.ToArray(); + } + private static byte[] ZLibDecompress(byte[] zlib) { using var input = new MemoryStream(zlib); From 0722f18212cab5144c6ad650409ea2e1e21b2b68 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 23:51:59 +0000 Subject: [PATCH 132/146] docs: say what a hidden end leaves the file and what a section's data holds beside a cut The architecture, the diagnostics reference and the threat model said data an endstream proven its own ends past white space decodes as read whole, every filter's end silenced: its first filter now decodes it again up to the run of white space and comments when it does not end it as its format says, its output's end the guard's where it may lack what the run holds, and the filters after a first that ended whole report as read whole. The known gap is narrowed to match. The lists of what is the file's beside a value a guard cut gain data that holds too few rows whatever the value holds, in ADR 34, the architecture, the concepts and the references; the reader limits reference no longer calls a subsection of no rows a fault. The search costs no longer claim about twice the file for three ways a deferred stream's white space crosses three times, the lazy reading page counts them as the others do, and the probe's comments, its regrowth among the known gaps, and the scope of a value's cut whatever the type are said as the code does them. Refs #224, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- ...-is-readable-and-the-readers-guards-are.md | 5 ++-- docs/architecture.md | 10 ++++--- docs/threat-model.md | 19 +++++++----- docs/website/docs/concepts/diagnostics.md | 3 +- docs/website/docs/concepts/lazy-reading.md | 5 ++-- docs/website/docs/concepts/reader-limits.md | 10 ++++--- docs/website/docs/concepts/validation.md | 3 +- docs/website/docs/reference/diagnostics.md | 12 ++++++-- docs/website/docs/reference/reader-limits.md | 10 ++++--- .../docs/reference/validation-rules.md | 2 +- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 29 ++++++++++++------- 11 files changed, 69 insertions(+), 39 deletions(-) diff --git a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md index 3ec2694f..70b9d449 100644 --- a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md +++ b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md @@ -100,7 +100,8 @@ bounded by `EndStreamSearchPasses`, four times the file for a document's searche constant with its reason rewritten: a valid file is read past the parser's window only by the search for the data of a cross-reference stream whose `/Length` the chain cannot read, by the search for the data of a stream whose `/Length` a guard cut or left out of the index, and by white space and comments after a declared length that run past a first -read — about twice the file at most; the proof that an `endstream` found under a hidden `/Length` is the stream's +read — about twice the file, three times over the white space a cross-reference stream's search takes as data that its +length, once read, is read on through; the proof that an `endstream` found under a hidden `/Length` is the stream's own reads 48 or 64 bytes past each, once, and the white space it crosses once more at most — the run before an `endobj` taken from the read-on that read it, a run a row names before the next header read once from the row —, about three times the file in all (amended on 2026-10-24). @@ -125,7 +126,7 @@ on 2026-10-24) —, stops the search, the data up to it then the guard's; a whole reference the cut left leads on, as read whole; a chain of filters ends there, its data left encoded; and a cross-reference section whose values lead to an object `MaxObjectLength` cut ends the -chain, nothing rebuilt — unless what the values read say makes it unreadable whatever the cut hides, the file's fault +chain, nothing rebuilt — unless what the values read, and data whose length the file gives, say makes it unreadable whatever the cut hides, the file's fault (amended on 2026-10-24) —, the older sections and their trailers the guard's to hide, and the guard reached once the chain is read. So does a cross-reference stream's `/Length` the chain could not read that leads to such an object, wherever only that length could say how far the data runs: no `endstream` comes before the next object or the end of the file, or diff --git a/docs/architecture.md b/docs/architecture.md index 0ee4016e..a5e94a02 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -142,7 +142,8 @@ separate types, separate severity scales, and vocabularies that never share a co through, as a load reads it; its chain of filters ends there, with the data left encoded; and the chain of cross-reference sections ends at that section, with nothing rebuilt and the older sections and their trailers the guard's to hide, wherever near its named offset the section is found — - unless what the values read say makes it unreadable whatever the cut hides, which is the file's fault. An + unless what the values read, and data whose length the file gives, say makes it unreadable whatever the cut + hides, which is the file's fault. An object `MaxObjectLength` cut there is loaded once the chain is read, which reaches the guard — thrown at opening under `ThrowOnLimit` —, for a value the chain keeps: not for a place it dropped as no section. A cross-reference stream's `/Length` the chain could not read, which leads to such an object, ends the chain so wherever only it @@ -151,9 +152,10 @@ separate types, separate severity scales, and vocabularies that never share a co of the length, the rows past it are the guard's — unless what follows the `endstream` the data was taken to proves it the stream's own, as in item 4, which leaves the rows the data lacks to the file. Such data, whose end a guard hides, is decoded again as data a guard cut, a group its end parts dropped and what a filter says of the - end silenced, no more rows held than the first decoding read; a decoding guard the first decoding reached stands. - Data an `endstream` proven its own ends past white space (item 4) decodes as read whole, what a filter says of - its end silenced. + end silenced; a decoding guard the first decoding reached stands, and no more rows are held than it read. + Data an `endstream` proven its own ends past white space or comments (item 4) decodes as read whole where its + first filter ends it as its format says; otherwise up to where that run starts, the filter's running out there + silenced, and where its output may lack what the run holds, its end the guard's, as cut. ### 3.2 The write path diff --git a/docs/threat-model.md b/docs/threat-model.md index d8d4d1b1..36abc77e 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -283,7 +283,7 @@ Known gaps: | A classic table of any length | Its window grows ×4 from 64 KB up to `MaxXRefSectionLength`, and the rows read are kept. A table the guard cuts, or whose rows run past a full index, keeps its trailer, found past the rows left unread by a search a megabyte at a time (`ReadTrailerPastRows`): comments are skipped, and anything else no row holds is the section's fault, reported, the search going on to the `trailer` keyword, so that its `/Encrypt` still refuses the document. The search starts at the start of the line the cut fell in, outside any comment and past the table's own keyword; it reads each megabyte once, every byte of it looked at before the next is read, however many comments or stray bytes it holds, and is not made for a place a section may have been moved to, a guess. The chain ends there, since an older section's rows would stand for the objects the unread rows define. What those rows place reads as null until the guard is raised | `ReaderLimitsTests.A_table_whose_keywords_the_bound_cuts_reports_the_bound`, `A_table_its_length_bound_cuts_keeps_its_trailer_and_the_encryption_it_names`, `A_table_its_length_bound_cuts_ends_the_chain_though_its_trailer_names_an_older_section`, `What_ends_the_rows_a_length_bound_cut_is_judged_as_it_is_where_none_did`, `A_table_a_length_bound_cuts_that_runs_to_the_end_of_the_file_has_no_trailer`, `A_table_its_length_bound_cuts_keeps_the_encryption_its_trailer_names_past_what_no_row_holds`, `A_table_cut_before_its_keyword_ends_keeps_its_trailer`, `Rows_past_a_full_index_are_not_read_and_the_trailer_past_them_is`, `A_cut_in_a_comment_among_the_rows_searches_for_the_trailer_from_the_comment_s_start`, `The_trailer_past_rows_left_unread_is_found_wherever_the_search_s_window_ends`, `A_trailer_past_rows_left_unread_that_its_own_bound_cuts_keeps_what_precedes_the_cut`; `DocumentReaderTests.The_search_for_the_trailer_past_rows_left_unread_reads_the_file_once`; `CrossReferenceChainTests.Reads_an_object_a_length_bound_cut_out_of_its_table_as_null_until_the_guard_is_lifted`, `A_section_found_near_where_prev_names_it_is_not_searched_past_the_rows_a_guard_left_unread`; `WindowEdgeTests.A_classic_table_reads_whole_wherever_its_window_edge_falls_near_its_trailer` | No: seeds stay under 120 KB | | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). A hybrid file's stream's own `/Prev`, which the chain does not follow, going on through the table's, is not read at all, nor are the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place: neither names a section missing nor ends the chain for a guard, and the rule on trailer values judges them as written, a reference they hold not followed ([#263]). An object the guard cut after a whole reference it holds hides nothing: the chain reads through it, as a load does, and loads it once the chain is read. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s`, `A_table_an_xrefstm_names_in_a_stream_s_place_has_its_own_links_left_unread`, `A_prev_written_as_a_reference_to_a_real_is_judged_through_it_only_where_the_reader_follows_it` | Yes | -| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); so is one that declared itself a section before a guard cut it — a table's `xref` keyword, `/Type /XRef`. A place that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is a guess dropped, the next place tried; where the chain names a section, a guard that cut the dictionary of what declares another type cut no section — a CIDFont's, which read whole no stream data follows —, and relocation is tried, while one that cut a value of a stream there leaves it the section the guard cut, whatever type it gives, as read whole it is the section. What makes a section unreadable whatever a value a guard cut holds is the file's fault, as read whole, judged before the cut is taken for the guard's (`PdfFileReader.FaultBesideCut`): a value past the cut that no section read so far places, which the sections a whole read reads before make the file's to lack; a `/W` that gives no three widths, or none past 0; rows numbered wrongly — an `/Index` that holds no pairs, whatever its elements, a subsection out of range, or wrong by the element read whatever the cut one holds, a `/Size` with no `/Index` that gives no count —, as far as the values read, written direct or through references, say: a value a guard cut says nothing, nor does what depends on it, the `/Size` an `/Index` so cut may stand beside | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_cross_reference_stream_whose_deferred_length_a_guard_cut_is_found_where_it_was_moved`, `A_cross_reference_stream_a_guard_cut_the_dictionary_of_after_its_type_is_the_section_where_it_was_moved`, `A_classic_table_a_guard_cut_the_rows_of_is_the_section_where_it_was_moved`, `A_stream_with_no_numbering_whose_width_a_guard_cut_is_no_section_the_guard_cut`, `A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild`, `What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_guard_cut_holds_is_the_file_s`, `A_place_a_section_may_have_moved_to_that_a_value_past_a_guard_s_cut_makes_unreadable_is_dropped`, `A_stream_whose_value_a_guard_cut_where_the_chain_names_a_section_is_the_section_the_guard_cut_whatever_its_type`, `A_cross_reference_stream_that_misspells_its_type_and_whose_value_a_guard_cut_is_the_section_the_guard_cut`, `A_section_a_value_s_cut_keeps_from_being_read_is_the_guard_s_where_nothing_read_says_it_is_wrong` | No | +| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); so is one that declared itself a section before a guard cut it — a table's `xref` keyword, `/Type /XRef`. A place that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is a guess dropped, the next place tried; where the chain names a section, a guard that cut the dictionary of what declares another type cut no section — a CIDFont's, which read whole no stream data follows —, and relocation is tried, while one that cut a value of a stream there leaves it the section the guard cut, whatever type it gives, as read whole it is the section. What makes a section unreadable whatever a value a guard cut holds is the file's fault, as read whole, judged before the cut is taken for the guard's (`PdfFileReader.FaultBesideCut`): a value past the cut that no section read so far places, which the sections a whole read reads before make the file's to lack; a `/W` that gives no three widths, or none past 0; rows numbered wrongly — an `/Index` that holds no pairs, whatever its elements, a subsection out of range, or wrong by the element read whatever the cut one holds, a `/Size` with no `/Index` that gives no count —, as far as the values read, written direct or through references, say: a value a guard cut says nothing, nor does what depends on it, the `/Size` an `/Index` so cut may stand beside; and data whose length the file gives, its filters written direct, decoded whole, that holds fewer rows than the counts read declare however few bytes the widths read leave a row, which lacks them whatever the cut hides: the section is there, not looked for nearby, and the index rebuilt | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_cross_reference_stream_whose_deferred_length_a_guard_cut_is_found_where_it_was_moved`, `A_cross_reference_stream_a_guard_cut_the_dictionary_of_after_its_type_is_the_section_where_it_was_moved`, `A_classic_table_a_guard_cut_the_rows_of_is_the_section_where_it_was_moved`, `A_stream_with_no_numbering_whose_width_a_guard_cut_is_no_section_the_guard_cut`, `A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild`, `What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_guard_cut_holds_is_the_file_s`, `A_place_a_section_may_have_moved_to_that_a_value_past_a_guard_s_cut_makes_unreadable_is_dropped`, `A_stream_whose_value_a_guard_cut_where_the_chain_names_a_section_is_the_section_the_guard_cut_whatever_its_type`, `A_cross_reference_stream_that_misspells_its_type_and_whose_value_a_guard_cut_is_the_section_the_guard_cut`, `A_section_a_value_s_cut_keeps_from_being_read_is_the_guard_s_where_nothing_read_says_it_is_wrong`, `A_section_whose_data_holds_fewer_rows_than_it_declares_whatever_a_value_a_guard_cut_holds_is_the_file_s_to_lack` | No | | A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, may stay within the bound — the group counted against what the filters before the last share as well, and no more rows held than the first decoding read —: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_decoding_guard_deferred_data_reached_is_reported_once_whatever_room_the_diagnostics_have`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `Data_whose_end_a_guard_hides_leaves_the_filters_after_a_dropped_group_the_room_its_direct_twin_s_have`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file`, `A_cross_reference_stream_the_chain_read_proves_the_endstream_of_the_older_one_before_it` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | @@ -305,7 +305,9 @@ Known gaps: - [#221] A cross-reference stream row of a reserved type is skipped, letting an older section's row stand where ISO 32000-1 reads the object as null. - [#156] An `/XRefStm` naming a section already read, or a classic table, leaves no record and no finding. -- [#47] Each section is read through a window of up to 64 KB, whatever its size. +- [#47] Each section is read through a window of up to 64 KB, whatever its size; and the probe that looks for a + classic table's `xref` keyword past white space an offset falls short by reads again from the offset as it grows, + eight times larger each time up to `MaxTrailerLength`: about nine times the run it crosses, once per section. - [#49] A rebuild reads a 64 KB window at every `trailer` keyword. - [#188] Relocating a section the chain cannot read can land on one it already read: a loop and a shift the file does not have are reported, or a hybrid table taken for its own `/XRefStm` loses what only the stream indexes. @@ -355,7 +357,7 @@ Known gaps: | A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | -| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`), so that a filter that runs out there — a Flate checksum short of a carriage return, or one that white space completes, an end marker missing — says nothing of the file, while the data decodes as read whole: white space parts no ASCII85 or ASCIIHex group, and no group is dropped nor a predictor's row undone for it. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `A_cross_reference_stream_named_before_white_space_proves_the_endstream_before_it_however_much`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `Data_that_ends_in_white_space_right_before_a_proven_endstream_ends_where_the_guard_hides`, `A_checksum_short_of_a_byte_white_space_before_a_proven_endstream_completes_is_no_fault_of_the_file`, `Predicted_rows_a_hidden_end_leaves_a_byte_short_are_undone_as_far_as_they_go`, `Lzw_data_that_lacks_its_end_code_before_a_proven_endstream_past_an_end_of_line_is_no_fault_of_the_file`, `An_ascii85_object_stream_that_lacks_its_end_marker_before_a_proven_endstream_decodes_as_read_whole`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | +| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space and comments the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`). Where the first filter ends the data as its format says, with nothing to report, it decodes as read whole; otherwise the filter decodes it again up to where that run starts, in time linear in the data: what it meets before is the file's — a checksum that disagrees with the bytes before it —, its running out there — a Flate checksum short, an end marker missing — says nothing of the file, and where its output may lack what the run holds — Flate that lost its tail, LZW or RunLength data without its end marker — the output's end is the guard's, cut, for the predictor, which undoes its last row as far as it goes, the filters after it and an object stream's last member; white space and comments decode to nothing in ASCII85 or ASCIIHex data, whose output is whole, and the filters after a first that ended whole decode as read whole, their faults the file's. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `A_cross_reference_stream_named_before_white_space_proves_the_endstream_before_it_however_much`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `Data_that_ends_in_white_space_right_before_a_proven_endstream_ends_where_the_guard_hides`, `A_checksum_short_of_a_byte_white_space_before_a_proven_endstream_completes_is_no_fault_of_the_file`, `A_checksum_that_disagrees_before_the_white_space_a_hidden_length_leaves_is_the_file_s`, `A_filter_after_one_that_ends_where_its_format_says_decodes_as_read_whole_whatever_hides_the_end`, `Predicted_rows_a_hidden_end_leaves_a_byte_short_are_undone_as_far_as_they_go`, `Lzw_rows_a_hidden_end_leaves_a_byte_short_are_undone_as_far_as_they_go`, `Lzw_data_that_lacks_its_end_code_before_a_proven_endstream_past_an_end_of_line_is_no_fault_of_the_file`, `White_space_and_comments_a_hidden_length_leaves_are_not_decoded_as_data_that_lacks_its_end_marker`, `An_object_stream_whose_last_code_a_hidden_end_leaves_a_byte_short_ends_where_the_guard_hides`, `An_ascii85_object_stream_that_lacks_its_end_marker_before_a_proven_endstream_decodes_as_read_whole`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | | Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, and the white space it crosses once more at most — the run before an `endobj` taken from the read-on that read it, a run a row names before the next header read once from the row —, about three times the file in all; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `DocumentReaderTests.Deferred_lengths_followed_by_long_white_space_have_it_read_on_once_however_their_data_ends`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_proofs_of_streams_white_space_follows_read_less_than_three_times_the_file`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | @@ -381,10 +383,13 @@ Known gaps: `endobj` or not, then `xref` or `startxref`, and goes on, ends there as the file's data, and its object as ending there: only the hidden length could tell, and a valid file's last stream before its table is told so ([#224]). - Accepted, with no issue: under a `/Length` a guard cut or left out of the index, data an `endstream` proven its own - ends past white space — the end-of-line the search leaves out before the keyword, above all — decodes with its end - the guard's: a Flate stream that lost its tail or its checksum, or whose checksum ends in that white space, or ASCII85, ASCIIHex, RunLength or LZW data without - its end marker, is not reported there, though a whole read reports it, by any filter of the stream's chain; only the - hidden length says whether that white space is the data's ([#224]). + ends past white space or comments — the end-of-line the search leaves out before the keyword, above all — decodes + with its end the guard's where its first filter does not end it as its format says: that filter's running out — a + Flate stream that lost its tail or its checksum, ASCII85, ASCIIHex, RunLength or LZW data without its end marker — is + not reported there, though a whole read reports it, nor what the filters after it meet of an end the guard's; a + checksum whose bytes reach into the run says nothing; and where the data's last bytes, read whole, look like white + space or a comment after white space, what a filter that ran out decodes from them is cut, the guard's. Only the hidden + length says whether that run is the data's ([#224]). - Accepted, with no issue: where the chain names a section, an object that declares another type and gives a `/W` before `MaxTrailerLength` cuts its dictionary is taken for no section, as a CIDFont's dictionary is read whole, no stream data after it, and the section is looked for nearby; a stream of that type, whose stream data the cut hides, diff --git a/docs/website/docs/concepts/diagnostics.md b/docs/website/docs/concepts/diagnostics.md index b2fc3fbe..ac379911 100644 --- a/docs/website/docs/concepts/diagnostics.md +++ b/docs/website/docs/concepts/diagnostics.md @@ -100,7 +100,8 @@ was read before it — though a damaged index can still serve another copy of an what was read first. A file crafted for it can still make what a stream holds depend on which object was asked for first. What the searches of one document read together is bounded: none starts once they have read four times the file's length. For a valid file, what they read — its cross-reference streams searched, its streams whose length a guard hid, -the white space after a declared length read on — comes to about twice the file at most, and the proofs that an `endstream` +the white space after a declared length read on — comes to about twice the file, three times over white space that is +both the data a search takes and the run a declared length is read on through, and the proofs that an `endstream` found under a hidden length is the stream's own, 48 or 64 bytes past each, once, and the white space they cross once more at most, to about three times the file in all; the searches of damaged streams read twice the file at most — 0.35 diff --git a/docs/website/docs/concepts/lazy-reading.md b/docs/website/docs/concepts/lazy-reading.md index af3a4aa4..80c0a565 100644 --- a/docs/website/docs/concepts/lazy-reading.md +++ b/docs/website/docs/concepts/lazy-reading.md @@ -145,8 +145,9 @@ through its own reads, the comments after it taken for white space, rather than backward. The search is made once for each stream, however often the stream is parsed again, and whether the object's `endobj` follows the declared length is read again only when a rebuild changes that length; no window grows to find an `endstream`, so that a stream's data never counts toward `MaxObjectLength`; and the searches of a document read four -times the file at most — a valid file's, about twice, for the white space after a declared length and the -cross-reference streams whose `/Length` the chain cannot read yet. What each case keeps, and reports, is in [Diagnostics](diagnostics.md#where-a-stream-ends). +times the file at most — a valid file's, about three times, for the white space after a declared length, the +cross-reference streams whose `/Length` the chain cannot read yet, and the streams whose `/Length` a guard hid, with the +proof that their `endstream` is their own. What each case keeps, and reports, is in [Diagnostics](diagnostics.md#where-a-stream-ends). When it is read or decoded, a stream works under the [limits](reader-limits.md) of the document it came from, however long after opening — so with `ThrowOnLimit` set, `GetRawBytes()` and `Decode()` can throw long after `Open` diff --git a/docs/website/docs/concepts/reader-limits.md b/docs/website/docs/concepts/reader-limits.md index 06463b5b..4ed114f6 100644 --- a/docs/website/docs/concepts/reader-limits.md +++ b/docs/website/docs/concepts/reader-limits.md @@ -63,8 +63,9 @@ then the file's, though white space before the keyword, which only the length sa ends to the guard as the data decodes: a filter that runs out there says nothing of the file. Decoding stops at a filter so named, or one whose parameters are, and leaves the rest of the data encoded, as past the most filters the reader runs; and a `/Prev` or an `/XRefStm` the reader follows that is so named, or a cross-reference stream whose values lead to an object a guard cut, ends the chain as a -section a guard cut does, wherever near its named offset the section is found — unless what the values read say makes -the section unreadable whatever the cut hides: a fault of the file, as read whole. +section a guard cut does, wherever near its named offset the section is found — unless what the values read, and data +whose length the file gives, say makes the section unreadable whatever the cut hides: a fault of the file, as read +whole. What a guard leaves out of the cross-reference index — the oldest sections of a long chain, the rows past a long table's bound, the objects past the most the index holds — is no damage either. Those objects read as null, unknown @@ -123,8 +124,9 @@ the object through a larger window, so that a stream's data — and the end-of-l counts toward its object's length. A document's searches read four times the file at most: a valid file is read past those bytes only for the data of a cross-reference stream whose `/Length` the chain cannot read yet, for the data of a stream whose `/Length` a guard cut or left out of the -index, and for white space and comments after a declared length, about twice the file, and for the proof that an -`endstream` found under a hidden `/Length` is the stream's own, about three times the file in all. Those stay internal +index, and for white space and comments after a declared length, about twice the file — three times over the white +space a cross-reference stream's search takes as data that its length, once read, is read on through —, and for the +proof that an `endstream` found under a hidden `/Length` is the stream's own, about three times the file in all. Those stay internal constants, since lifting them would let nothing more be read, and each says where it is declared why no valid file reaches it. diff --git a/docs/website/docs/concepts/validation.md b/docs/website/docs/concepts/validation.md index 21e0ada8..fb0949e8 100644 --- a/docs/website/docs/concepts/validation.md +++ b/docs/website/docs/concepts/validation.md @@ -157,7 +157,8 @@ information, that it could not check it whole, rather than calling the file brok limit lets it check the rest. The same holds of a value that leads to such an object: a cross-reference stream whose `/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, leads to one is said not checked rather than malformed — so is one whose `/Length`, which the chain could not read, leads to one where only that length could say -where its data ends —, unless what the values read say makes it unreadable whatever the guard hid, which is the file's; +where its data ends —, unless what the values read, and data whose length the file gives, say makes it unreadable whatever the guard hid, +which is the file's; a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain there rather than naming a section not found; and an object stream whose `/Filter` or `/DecodeParms` leads to one, or to an object a limit left out of the index, is left encoded, the objects it holds said not checked rather than lost diff --git a/docs/website/docs/reference/diagnostics.md b/docs/website/docs/reference/diagnostics.md index 3f1b0c4d..4db2c4d9 100644 --- a/docs/website/docs/reference/diagnostics.md +++ b/docs/website/docs/reference/diagnostics.md @@ -102,7 +102,7 @@ Codes are stable: they are part of the public contract, because callers filter o | `xref.rebuilt` | The index was rebuilt by scanning the whole file | | `xref.offset-adjusted` | An object, or a cross-reference section the chain names, was not where the file said, and was found nearby | | `xref.section-missing` | A cross-reference section `/Prev` or `/XRefStm` names holds nothing that reads as a section, nor does any place nearby, lies outside the file, or is named by a value that is not an offset; what only it indexed is found by rebuilding the index when it is asked for. Reported at the offset the section was named at, or, when that offset lies outside the file or the value is not an offset, at the section whose trailer named it. A `/Prev` or `/XRefStm` the reader follows — not a hybrid file's cross-reference stream's own `/Prev`, nor the links of a table an `/XRefStm` names, which it leaves unread —, written as a reference that leads to an object one of the reader's limits cut, names what the guard hides: no section is reported missing, nothing is rebuilt, the chain ends there, and the guard is reported once the chain is read | -| `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, data or — with no `/Index` — `/Size` cannot give rows, or whose `/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, is a reference no section read before it places where it can be read without loading it, or whose `/Length` is such a reference, or leads to one through others, and no `endstream` follows its data before the next object the sections read so far place, or before the end of the file, or none was looked for once the document's searches for `endstream` read as much of the file as they may — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message. A value that names an object one of the reader's limits cut, directly or through a chain of references — a `/Length` whose data no `endstream` ends before that stop among them —, is the guard's, not the file's: the section is not reported, nor looked for nearby, the chain ends there, nothing is rebuilt, what only that section and the older ones index reads as null, and the guard is reported once the chain is read — but for what makes the section unreadable whatever the value holds, which is the file's and reported here, as read whole: a value past it no section read so far places, a `/W` that gives no three widths or none past 0, or rows numbered wrongly as far as the values read say — a subsection wrong by the element read whatever the cut one holds among them | +| `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, data or — with no `/Index` — `/Size` cannot give rows, or whose `/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, is a reference no section read before it places where it can be read without loading it, or whose `/Length` is such a reference, or leads to one through others, and no `endstream` follows its data before the next object the sections read so far place, or before the end of the file, or none was looked for once the document's searches for `endstream` read as much of the file as they may — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message. A value that names an object one of the reader's limits cut, directly or through a chain of references — a `/Length` whose data no `endstream` ends before that stop among them —, is the guard's, not the file's: the section is not reported, nor looked for nearby, the chain ends there, nothing is rebuilt, what only that section and the older ones index reads as null, and the guard is reported once the chain is read — but for what makes the section unreadable whatever the value holds, which is the file's and reported here, as read whole: a value past it no section read so far places, a `/W` that gives no three widths or none past 0, or rows numbered wrongly as far as the values read say — a subsection wrong by the element read whatever the cut one holds among them —, or data whose length the file gives, decoded whole, that holds fewer rows than the counts read declare however few bytes the widths read leave a row, a section then not looked for nearby | | `xref.chain-cycle` | The chain of previous sections looped: a `/Prev` or `/XRefStm` names an offset the chain has already named, whether it read a section there or not. Reported at that offset, or, when it lies outside the file, at the section whose trailer named it | | `xref.entry-out-of-range` | An object's entry places it outside the file — past its end, before its start, or past what a `long` holds once the header's offset is added —; reported with no position, the offset in the message, exactly. A section named outside the file is `xref.section-missing`'s, or the rebuild's when `startxref` names it | | `stream.length-invalid` | A stream's `/Length` is not where its data ends: its `endstream` lies elsewhere and ends the data; or the `/Length` is no length — absent, not a non-negative integer, or naming an object the file lacks or that could not be read — and the `endstream` ends the data; or no `endstream` lies before the next object or the end of the file, no `endobj` of its object ends the data, and the data is the declared length, when it ends before that stop, or runs up to the next object; or the document's searches for `endstream` have read as much of the file as they may, and the data is taken without a search — but where a guard hid the `/Length`, whose data is then the guard's. A `/Length` that names an object a guard cut, or left out of the index, is not judged, though a missing `endstream` is reported all the same. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case. A cross-reference stream whose `/Length` the chain could not read, its data taken up to its first `endstream`, has it read once the chain is, unless a guard cut that data before every row the section declares — data that holds them all is checked whatever a guard did —: a `/Length` that is no length, or that no `endstream` confirms, is reported as any stream's, and one an `endstream` past the one the chain stopped at confirms is reported with the rows the chain left unread. Where the section's own row for that `/Length` lies among the rows the data holds, and gives a length an `endstream` confirms, the data is taken again to it before the rows are read, and nothing is reported; nor is anything of a `/Length` a guard cut or left out of the index | @@ -257,8 +257,14 @@ data within `MaxStreamLength`, or its object within `MaxObjectLength`, or a filt `MaxDecodedStreamLength`. Of data the reader cut, no lost tail, missing checksum or missing end marker is reported, a hexadecimal digit without its pair or an ASCII85 group short of five is dropped rather than completed, and a predictor undoes the part of its last row the data holds rather than report rows the parameters describe as longer than the data. -Nor is any reported of data whose end a guard hides — its `endstream` proven its own past white space that only a -`/Length` a guard cut or left out of the index says is the data's or not —, which decodes as read whole. +Nor is the end of data whose end a guard hides — its `endstream` proven its own past white space or comments that only +a `/Length` a guard cut or left out of the index says are the data's or not — reported. Where its first filter ends it +as its format says, with nothing to report, the data decodes as read whole; otherwise the filter decodes it up to where +that run starts: what it meets before is the file's and reported — a checksum that disagrees with the bytes before it +among it —, its running out there — a lost tail, a checksum short, a missing end marker — is not, and where its output +may lack what the run holds — Flate that lost its tail, LZW or RunLength data without its end marker — the output's end +is cut, as above, for the predictor, the filters after it and an object stream's members. The filters after a first that +ended whole decode as read whole, and report as read whole. ```text Warning filter.failed at 59534: A Flate stream ends before its data does; what decoded before the end was kept. diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index 7859fafe..08b29cd1 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -144,10 +144,12 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. a CIDFont's, which read whole no stream data follows —, while one that cut a value of a stream there leaves it the section the guard cut, whatever type it gives, as read whole it is the section. What makes a section unreadable whatever a value a guard cut holds is the file's fault, as read whole: a value past the cut that no section read so far places, a - `/W` that gives no three widths or none past 0, or rows numbered wrongly — an `/Index` that holds no pairs, a - subsection that numbers no object or numbers past the object numbers, by both its elements or by the one read whatever - the cut one holds, a `/Size` with no `/Index` that gives no count —, as far as the values read, written direct or - through references, say. The rows a cross-reference stream's data lacks, + `/W` that gives no three widths or none past 0, rows numbered wrongly — an `/Index` that holds no pairs, a + subsection whose elements are no integers, whose count is below 0, or that numbers past the object numbers, by both + its elements or by the one read whatever the cut one holds, a `/Size` with no `/Index` that gives no count —, as far as + the values read, written direct or through references, say, or data whose length the file gives, decoded whole, that + holds fewer rows than the counts read declare however few bytes the widths read leave a row: such a section is not + looked for nearby. The rows a cross-reference stream's data lacks, where its `/Length` so leads and its data was taken to the first `endstream`, are the guard's too, unless what follows that `endstream` proves it the stream's own, as above: they are then the file's to lack, and the index is rebuilt. An object the chain read through a reference the cut left whole is loaded once the chain is read only for a value it diff --git a/docs/website/docs/reference/validation-rules.md b/docs/website/docs/reference/validation-rules.md index d99e4a24..b980e1f5 100644 --- a/docs/website/docs/reference/validation-rules.md +++ b/docs/website/docs/reference/validation-rules.md @@ -51,7 +51,7 @@ References are to ISO 32000-1:2008. | Rule | Severity | Meaning | Reads | |---|---|---|---| -| `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]); or a cross-reference stream whose `/Length` is such a reference, or leads through references to an object no section read before it places, and whose data no `endstream` ends before the next object the sections read so far place, or the end of the file — or was not searched for one, the document's searches for `endstream` having read as much of the file as they may ([#220]). Rows that spell `endstream` end such a stream's data there, and the section lacks the rows past them, unless the section's own row for that `/Length` — the row the reading of its rows keeps, one of a reserved type ignored —, among the rows before them, places an object that gives a longer length an `endstream` confirms: the data is then taken to it. A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. Nor is a cross-reference stream whose value the rows are read with leads to an object a limit cut: the section counts as cut there, the chain ends at it, nothing is rebuilt, and `xref.checked-in-part` says so ([#263]) — but for what makes it unreadable whatever the value holds, as read whole: a value past it no section read so far places, a `/W` that gives no three widths or none past 0, or rows numbered wrongly as far as the values read say — a subsection wrong by the element read whatever the cut one holds among them —, whatever type the stream gives. Nor is one whose `/Length`, which the chain could not read, leads to an object `MaxObjectLength` cuts, where only that length could say where the data ends: with no `endstream` before the next object or the end of the file, the section counts as cut, as above; when its rows spell one, the integer the cut left of that object — whether the section's own row places it, or a newer section does — is where an `endstream` is looked for, the data taken to it when one confirms it, and otherwise the rows past the spelled one the limit's, the section counting as cut there, as above ([#224]). | Each section of the chain | +| `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]); or a cross-reference stream whose `/Length` is such a reference, or leads through references to an object no section read before it places, and whose data no `endstream` ends before the next object the sections read so far place, or the end of the file — or was not searched for one, the document's searches for `endstream` having read as much of the file as they may ([#220]). Rows that spell `endstream` end such a stream's data there, and the section lacks the rows past them, unless the section's own row for that `/Length` — the row the reading of its rows keeps, one of a reserved type ignored —, among the rows before them, places an object that gives a longer length an `endstream` confirms: the data is then taken to it. A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. Nor is a cross-reference stream whose value the rows are read with leads to an object a limit cut: the section counts as cut there, the chain ends at it, nothing is rebuilt, and `xref.checked-in-part` says so ([#263]) — but for what makes it unreadable whatever the value holds, as read whole: a value past it no section read so far places, a `/W` that gives no three widths or none past 0, or rows numbered wrongly as far as the values read say — a subsection wrong by the element read whatever the cut one holds among them —, or data whose length the file gives, decoded whole, that holds fewer rows than the counts read declare however few bytes the widths read leave a row, whatever type the stream gives. Nor is one whose `/Length`, which the chain could not read, leads to an object `MaxObjectLength` cuts, where only that length could say where the data ends: with no `endstream` before the next object or the end of the file, the section counts as cut, as above; when its rows spell one, the integer the cut left of that object — whether the section's own row places it, or a newer section does — is where an `endstream` is looked for, the data taken to it when one confirms it, and otherwise the rows past the spelled one the limit's, the section counting as cut there, as above ([#224]). | Each section of the chain | | `xref.section-not-found` | Error | A section a `/Prev` or an `/XRefStm` names is neither there nor within 512 bytes of it, or what names it is not an offset — tiff2pdf writes `/Prev 576066 0 R`. The objects only it indexes are missing from the index. The section `startxref` names is `file.startxref-wrong`'s. A `/Prev` or an `/XRefStm` the reader follows, written as a reference that leads to an object `MaxObjectLength` cuts, names no section missing: the chain ends there for the limit, what it names unread, which `xref.checked-in-part` says ([#263]). | Each section of the chain | | `xref.section-shifted` | Warning | A section a `/Prev` or an `/XRefStm` names starts a few bytes from where it is named, and the reader found it there. | Each section of the chain | | `xref.chain-loop` | Error | The chain names a section it has already read, or an offset it has already named where it read none, in the file or outside it: the reader stops there, and the section the chain should have gone on to is unknown, what only it indexed found, if at all, by rebuilding the index — as for `xref.section-not-found`. | The chain | diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 096ed2b0..2697376b 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -37,7 +37,11 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// What the codes of the faults of the object syntax start with. private const string SyntaxCodePrefix = "syntax."; - /// How much of a section is read to tell a classic table from a cross-reference stream. + /// + /// How much of a section is read first to tell a classic table from a cross-reference stream: a read that holds + /// white space and comments alone, or a first token its edge may cut, is read again eight times larger, as far as a + /// trailer's window reaches. + /// internal const int XRefProbeLength = 32; /// @@ -151,14 +155,17 @@ internal sealed class PdfFileReader : IPdfObjectSource, IPdfStreamDataProvider /// header its data spells, which leaves the data up to it to the guard; and white space after a declared length /// that runs past a first read is read on once for each place, and past an endstream once for each keyword /// however many places lead to it (#174). The data of different streams lies apart, and so do the runs of white - /// space after them: together those reads cover the file once, and one read past each — about twice the file at - /// most. The fourth adds a little to each stream of the second way: the proof that an endstream found is the + /// space after them: together those reads cover the file once, and one read past each — about twice the file —, but + /// where one run is two ways' at once: the white space before the endstream of a cross-reference stream whose + /// /Length the chain cannot read is the data its search takes and the run its length, once read, is read on + /// through, about three times. The fourth adds a little to each stream of the second way: the proof that an endstream found is the /// stream's own (#224) reads past it to its object's end and what follows, three first reads of /// bytes, and a fourth past a row that names the white space before the next object's /// header, each once however often the stream is parsed — 48 or 64 bytes, about what the smallest such stream spans /// with its header. White - /// space it crosses adds to that: the run before the object's endobj it takes from the read-on that read it, - /// the run after it it doubles through, as the read-on does, and the run a row names before the next header it + /// space it crosses adds to that: the run before the object's endobj, or before what follows the keyword where + /// no endobj does, it takes from the read-on that read it, the run after an endobj it doubles through, as + /// the read-on does, and the run a row names before the next header it /// reads once from there, a first read reaching the header. No run is read more than three times, and the file /// about three times at most in all, within the four the searches may read. Every other search is of an invalid /// stream, whose endstream does not follow its declared length. Each reads from a stream's data to the first @@ -3283,7 +3290,7 @@ private XRefSectionState TryReadXRefSection( // A probe that holds white space and comments alone, or a first token its edge may cut, is read on as far as a // trailer's window reaches, past which a stream's header is not looked for either: the keyword is the table's - // however far the offset falls short of it. + // wherever within that window the offset falls short of it. var probeLength = XRefProbeLength; while (true) @@ -3837,7 +3844,8 @@ private bool ReadSubsection(ref PdfLexer lexer, int first, int count, XRefSectio /// An object that declares itself a cross-reference stream — /Type /XRef, or a /W, beside no other /// /Type at a place the section may have been moved to — and cannot be read is a malformed section; any /// other object there, or none, is no section at all, and a guard that cut the dictionary of what declares another - /// type cut no section, though one that cut a value of a stream there leaves it the section, as read whole. What + /// type cut no section, though one that cut a value of a stream with a /W where the chain names the section + /// leaves it the section, as read whole — a place it may have been moved to takes no stream of another type. What /// makes it unreadable whatever a value a guard cut holds is the file's fault, as read whole /// (). At a place the section may have been moved to (), /// nothing such an object's reading met is reported or recorded — its search for an endstream aside, which is @@ -3908,7 +3916,7 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect section.Kind = XRefSectionKind.Stream; section.StreamObjectNumber = number; - // The probe that looked for the xref keyword says where the header lies only within its first bytes. + // The header lies where the object read starts, past however much white space the offset falls short by. section.Padding = (int)(header - absolute); section.TrailerPosition = absolute; section.TypedXRef = dictionary.IsOfType(PdfName.XRef); @@ -4447,8 +4455,9 @@ private static bool DeclaresCrossReferenceStream(PdfObject value) => /// /// Takes a section a value's cut keeps from being read for the guard's: the section counts as cut, the chain ends there /// and nothing is rebuilt for it (ADR 34), and the object the guard cut is loaded once the chain is read, which reaches - /// the guard (#263). Whatever /Type the stream gives, as where the chain names it a stream with a /W is read - /// as the section whole: the value hides whether its rows are. + /// the guard (#263). Where the chain names the section, whatever /Type the stream gives, as a stream with a + /// /W is read there as the section whole — a place it may have been moved to takes no stream of another type —: + /// the value hides whether its rows are, but for what finds wrong whatever it holds. /// /// The section, whose fault names the value. /// The object the guard cut that the value leads to. From a8077d8863f555784f62d7216e4a9d54098f53a2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 05:12:59 +0000 Subject: [PATCH 133/146] fix: read a section whose data holds too few rows beside a cut value as read whole A cross-reference stream beside a value a guard cut whose data cannot hold the rows declared was taken for a section that cannot be read: the chain ended there, its trailer and the older ones were not merged, so /Info, /ID or /Root only they give were lost, the validator judged no trailer value of it, and at a place a few bytes from the offset named it was passed over for an older revision's stream nearby, which was read in its place. Read whole, it is read and lacks rows. It now is so under the guard too: its trailer merged, the chain going on to its /Prev, the index rebuilt, and taken where it was moved to. A width past what an int counts no longer wraps the smallest row a section's widths allow, and a lone byte or row is said as one. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 32 +++++-- .../IO/XRef/XRefSectionRecord.cs | 5 +- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 95 +++++++++++++++---- 3 files changed, 104 insertions(+), 28 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 2697376b..9da562d2 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -2913,8 +2913,7 @@ private bool TryReadXRefChain(long startOffset) } // A section a guard cut is where it was named, and is not looked for elsewhere: what lies near it would - // stand for it. So is one that lacks rows whatever a value a guard cut holds, as read whole. - if (section.CutByLimit || section.Incomplete || !TryRelocateXRefSection(offset, section, out previous, out hybrid)) + if (section.CutByLimit || !TryRelocateXRefSection(offset, section, out previous, out hybrid)) { ReportLostSection(section, naming, offset, namedFrom); break; @@ -2968,7 +2967,7 @@ private bool TryReadXRefChain(long startOffset) _structure.Add(stream); if (TryReadXRefSection(stream.Offset, stream, out _, out _) != XRefSectionState.Read && - (stream.CutByLimit || stream.Incomplete || !TryRelocateXRefSection(hybrid, stream, out _, out _))) + (stream.CutByLimit || !TryRelocateXRefSection(hybrid, stream, out _, out _))) { ReportLostSection(stream, "/XRefStm", hybrid, section.Offset); } @@ -3989,13 +3988,25 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect return XRefSectionState.Malformed; } - // Data whose extent the file gives that cannot hold the rows the values read declare lacks rows whatever the cut - // value holds, as read whole: the section is there, the file's to lack them, and not looked for nearby. if (FaultBesideCut(dictionary, deferred is null && !xrefStream.Data.CutByGuard && !xrefStream.Data.EndHidden ? xrefStream : null, out var lacking) is { } file) { section.Fault = file; - section.Incomplete = lacking; - return XRefSectionState.Malformed; + + if (!lacking) + { + return XRefSectionState.Malformed; + } + + // Data whose extent the file gives that cannot hold the rows the values read declare lacks rows whatever the + // cut value holds: the section is read as read whole, the file's to lack them — its trailer merged, the chain + // going on, the index rebuilt —, though the rows it holds, which the cut value is read with, are not read. + _indexIncomplete = true; + section.Incomplete = true; + _xref.MergeTrailer(dictionary); + string? hiddenPrevious = null; + previous = section.NamedBy == "/XRefStm" ? -1 : SectionOffset(dictionary, PdfName.Prev, absolute, out hiddenPrevious); + section.ChainHiddenByGuard = hiddenPrevious; + return XRefSectionState.Read; } return CutByValue(section, cut); @@ -4520,8 +4531,9 @@ private XRefSectionState CutByValue(XRefSectionRecord section, PdfObjectId cut) anyWidth |= width > 0; - // A width is within an int (WidthFault), and three within a long. - row += width.GetValueOrDefault(); + // A width past what an int counts makes a row longer than any data decodes to: three so narrowed are within + // a long. + row += Math.Min(width.GetValueOrDefault(), int.MaxValue + 1L); } if (known && !anyWidth) @@ -4591,7 +4603,7 @@ private XRefSectionState CutByValue(XRefSectionRecord section, PdfObjectId cut) lacking = true; return string.Create( CultureInfo.InvariantCulture, - $"its data, {most.Length:N0} bytes decoded, holds at most {most.Rows:N0} of the {declared:N0}{(exact ? string.Empty : " or more")} rows its /Index and /Size declare"); + $"its data, {most.Length:N0} {(most.Length == 1 ? "byte" : "bytes")} decoded, holds at most {most.Rows:N0} of the {declared:N0}{(exact ? string.Empty : " or more")} {(declared == 1 && exact ? "row" : "rows")} its /Index and /Size declare"); } /// diff --git a/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs b/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs index 07d5911c..06de2ebe 100644 --- a/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs +++ b/src/AdCodicem.Pdf/IO/XRef/XRefSectionRecord.cs @@ -65,8 +65,9 @@ public XRefSectionRecord(string namedBy, long namedOffset, long namedFrom) public string? RefusedRow { get; set; } /// - /// Gets a value indicating whether a cross-reference stream holds fewer rows than it declares: read, the rows it holds - /// placing objects, or left unread beside a value a guard cut, whose data holds too few whatever the value holds. + /// Gets a value indicating whether a cross-reference stream holds fewer rows than it declares: the rows it holds place + /// objects, but where a value they are read with is one a guard cut, beside which the data holds too few whatever the + /// value holds. /// public bool Incomplete { get; set; } diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 96008177..7718095b 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -3028,7 +3028,6 @@ public void A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_n [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [/x 5 0 R]", "Index element", null)] [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [5 0 R 3000000000]", "Index element", null, false, "its /Index gives a subsection of 3,000,000,000 rows, more than object numbers 0 to 2147483647 hold")] [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [5 0 R 2147483649]", "Index element", null, false, "its /Index gives a subsection of 2,147,483,649 rows, more than object numbers 0 to 2147483647 hold")] - [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [5 0 R 2147483648]", "Index element", null, false, "its data, 56 bytes decoded, holds at most 8 of the 2,147,483,648 rows its /Index and /Size declare")] [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [-1 5 0 R]", "Index element", null, false, "its /Index gives a subsection from object -1, outside object numbers 0 to 2147483647")] [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [2147483648 5 0 R]", "Index element", null, false, "its /Index gives a subsection from object 2147483648, outside object numbers 0 to 2147483647")] [InlineData("/Type /XRef /Size 8 /W [1 4 2] /Index [9 0 R -1] /Length 5 0 R", "Length", "cut:0")] @@ -3187,19 +3186,27 @@ public void A_stream_whose_value_a_guard_cut_where_the_chain_names_a_section_is_ } [Theory] - [InlineData("<< /Type /Font /W 5 0 R /Size 8 /Length 3 >>\nstream\nabc\nendstream", "[1 4 2]", "holds at most 3 of the 8 rows")] - [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Length 3 >>\nstream\nabc\nendstream", "[1 4 2]", "holds at most 3 of the 8 rows")] - [InlineData("<< /Type /Xref /W [1 5 0 R 2] /Size 8 /Length 3 >>\nstream\nabc\nendstream", "4", "holds at most 1 of the 8 rows")] - [InlineData("<< /Type /XRef /W [1 4 2] /Index [5 0 R 8] /Length 3 >>\nstream\nabc\nendstream", "0", "holds at most 0 of the 8 rows")] - [InlineData("<< /Type /XRef /W [1 4 2] /Index [5 0 R 1 9 5 0 R] /Length 3 >>\nstream\nabc\nendstream", "0", "holds at most 0 of the 1 or more rows")] - [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Filter /FlateDecode /Length {length} >>\nstream\n{flate}\nendstream", "[1 4 2]", "its data, 5 bytes decoded, holds at most 5 of the 8 rows")] + [InlineData("<< /Type /Font /W 5 0 R /Size 8 /Length 3 >>\nstream\nabc\nendstream", "[1 4 2]", "/Prev", "holds at most 3 of the 8 rows")] + [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Length 3 /Info 8 0 R /ID [<01> <02>] >>\nstream\nabc\nendstream", "[1 4 2]", "/Prev", "holds at most 3 of the 8 rows")] + [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Length 3 >>\nstream\nabc\nendstream", "[1 4 2]", "/Prev+2", "holds at most 3 of the 8 rows")] + [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Length 3 >>\nstream\nabc\nendstream", "[1 4 2]", "/XRefStm", "holds at most 3 of the 8 rows")] + [InlineData("<< /Type /Xref /W [1 5 0 R 2] /Size 8 /Length 3 >>\nstream\nabc\nendstream", "4", "/Prev", "holds at most 1 of the 8 rows")] + [InlineData("<< /Type /XRef /W [1 4 2] /Index [5 0 R 8] /Length 3 >>\nstream\nabc\nendstream", "0", "/Prev", "its data, 3 bytes decoded, holds at most 0 of the 8 rows")] + [InlineData("<< /Type /XRef /W [1 4 2] /Index [5 0 R 1] /Length 1 >>\nstream\na\nendstream", "0", "/Prev", "its data, 1 byte decoded, holds at most 0 of the 1 row its")] + [InlineData("<< /Type /XRef /W [1 4 2] /Index [5 0 R 1 9 5 0 R] /Length 3 >>\nstream\nabc\nendstream", "0", "/Prev", "holds at most 0 of the 1 or more rows")] + [InlineData("<< /Type /XRef /W [1 4 2] /Index [5 0 R 2147483648] /Length 56 >>\nstream\n{56}\nendstream", "0", "/Prev", "holds at most 8 of the 2,147,483,648 rows")] + [InlineData("<< /Type /XRef /W [1 9223372036854775807 5 0 R] /Size 8 /Length 56 >>\nstream\n{56}\nendstream", "0", "/Prev", "holds at most 0 of the 8 rows")] + [InlineData("<< /Type /XRef /W [4611686018427387904 4611686018427387904 5 0 R] /Size 8 /Length 56 >>\nstream\n{56}\nendstream", "0", "/Prev", "holds at most 0 of the 8 rows")] + [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Filter /FlateDecode /Length {length} >>\nstream\n{flate}\nendstream", "[1 4 2]", "/Prev", "its data, 5 bytes decoded, holds at most 5 of the 8 rows")] public void A_section_whose_data_holds_fewer_rows_than_it_declares_whatever_a_value_a_guard_cut_holds_is_the_file_s_to_lack( - string six, string five, string fault) + string six, string five, string naming, string fault) { // As above, object 6's data, whose length it gives, holds fewer rows than its /Size or the /Index counts read declare, - // however few bytes the widths read and the one cut give a row: read whole, it lacks rows, the file's, and the index - // is rebuilt. It was taken for the guard's: the chain ended, nothing was rebuilt and the catalog was lost (#263). It - // is not looked for nearby either, where an older revision's stream would stand for it. + // however few bytes the widths read and the one cut give a row — a width past what data holds counted as such —: + // read whole, it lacks rows, the file's, its trailer merged, the chain going on, and the index rebuilt. It was taken + // for the guard's: the chain ended, nothing was rebuilt and the catalog was lost (#263); then for a section that + // cannot be read, whose trailer was lost, and which, where the chain names a place a few bytes off, was passed over + // for an older revision's stream nearby. Named by /XRefStm it is read as named. using var output = new MemoryStream(); using (var zlib = new ZLibStream(output, CompressionLevel.NoCompression, leaveOpen: true)) @@ -3210,7 +3217,8 @@ public void A_section_whose_data_holds_fewer_rows_than_it_declares_whatever_a_va var flate = output.ToArray(); six = six .Replace("{flate}", Encoding.Latin1.GetString(flate), StringComparison.Ordinal) - .Replace("{length}", Invariant($"{flate.Length}"), StringComparison.Ordinal); + .Replace("{length}", Invariant($"{flate.Length}"), StringComparison.Ordinal) + .Replace("{56}", new string('0', 56), StringComparison.Ordinal); var built = ChainFiles.UnderAnUpdate( "/Type /XRef /Size 8 /W [1 4 2] /Length {length}", five + new string(' ', 1100), @@ -3222,28 +3230,83 @@ public void A_section_whose_data_holds_fewer_rows_than_it_declares_whatever_a_va var text = Encoding.Latin1.GetString(built); var stream = text.IndexOf("\n7 0 obj", StringComparison.Ordinal) + 1; var sixAt = text.IndexOf("\n6 0 obj", StringComparison.Ordinal) + 1; - var file = Encoding.Latin1.GetBytes(text.Replace(Invariant($"/Prev {stream} "), Invariant($"/Prev {sixAt} "), StringComparison.Ordinal)); + var named = naming switch + { + "/Prev+2" => Invariant($"/Prev {sixAt + 2} "), + "/XRefStm" => Invariant($"/XRefStm {sixAt} "), + _ => Invariant($"/Prev {sixAt} "), + }; + var file = Encoding.Latin1.GetBytes(text.Replace(Invariant($"/Prev {stream} "), named, StringComparison.Ordinal)); + string[] wholeCodes; + string[] wholeRules; + string[] wholeKeys; using (var whole = PdfDocument.Open(file)) { whole.WasRepaired.Should().BeTrue(); whole.Reader.Structure.Sections[^1].Incomplete.Should().BeTrue(); + wholeCodes = [.. whole.Diagnostics.Select(entry => entry.Code)]; + wholeRules = [.. new PdfValidator().Validate(whole).Findings.Select(finding => finding.RuleId).Distinct().Order()]; + wholeKeys = [.. whole.Trailer.Keys.Select(key => key.Value).Order()]; } using var document = PdfDocument.Open(file, ObjectGuard); var section = document.Reader.Structure.Sections[^1]; section.Offset.Should().Be(sixAt); - section.State.Should().Be(XRefSectionState.Malformed); + section.State.Should().Be(naming == "/Prev+2" ? XRefSectionState.Relocated : XRefSectionState.Read); section.CutByLimit.Should().BeFalse(); section.Incomplete.Should().BeTrue(); section.Fault.Should().Contain(fault); document.WasRepaired.Should().BeTrue(); document.GetObject(new PdfObjectId(1)).AsDictionary().Required().IsOfType(PdfName.Catalog).Should().BeTrue(); + document.Trailer.Keys.Select(key => key.Value).Order().Should().Equal(wholeKeys); // The rebuild reaches the object the guard cut, as it reaches every object. - document.Diagnostics.Select(entry => entry.Code).Should().Equal( - PdfDiagnosticCodes.XRefSectionUnreadable, PdfDiagnosticCodes.XRefRebuilt, PdfDiagnosticCodes.LimitObject); + document.Diagnostics.Select(entry => entry.Code).Where(code => code != PdfDiagnosticCodes.LimitObject).Should().Equal(wholeCodes); + new PdfValidator().Validate(document).Findings.Select(finding => finding.RuleId).Distinct().Order().Should().Equal(wholeRules); + } + + [Fact] + public void Data_a_length_the_chain_cannot_read_yet_leaves_to_a_spelled_endstream_says_nothing_of_the_rows_it_holds() + { + // Stream 7's /W names object 5, which a newer table places and MaxObjectLength cuts, and its /Length object 8, which + // only its own rows place: its data runs to the first endstream, which its rows spell after six of eight. That data + // says nothing of how many rows the section holds, which the length, once read, may take further — read whole, it + // is taken again to its 56 bytes and every row is read —: the section is the guard's, not the file's to lack rows + // the data taken so far cannot hold at six bytes a row (#220, #263). + var file = new HandWrittenFile("", "1.5"); + var one = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + var two = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + var three = file.Object(3, Page); + var five = file.Object(5, "1" + new string(' ', 1100)); + var eight = file.Object(8, "56"); + var at = file.Position; + + static string Row(int type, long offset, int generation) => string.Concat( + (char)type, (char)((offset >> 24) & 0xFF), (char)((offset >> 16) & 0xFF), (char)((offset >> 8) & 0xFF), + (char)(offset & 0xFF), (char)(generation >> 8), (char)(generation & 0xFF)); + + var rows = Row(0, 0, 65535) + Row(1, one, 0) + Row(1, two, 0) + Row(1, three, 0) + Row(1, eight, 0) + Row(1, at, 0) + + "\nendstream\0\0\0\0"; + file.Raw(Invariant($"7 0 obj\n<< /Type /XRef /W [5 0 R 4 2] /Index [0 4 8 1 7 1 20 2] /Root 1 0 R /Length 8 0 R >>\nstream\n{rows}\nendstream\nendobj\n")); + file.Raw("%" + new string('-', 1200) + "\n"); + var table = file.Position; + file.Raw(Invariant($"xref\n0 1\n0000000000 65535 f \n5 1\n{five:D10} 00000 n \ntrailer\n<< /Size 22 /Root 1 0 R /Prev {at} >>\nstartxref\n{table}\n%%EOF\n")); + + using (var whole = PdfDocument.Open(file.Bytes)) + { + whole.WasRepaired.Should().BeFalse(); + whole.Reader.Structure.Sections[^1].Incomplete.Should().BeFalse(); + whole.GetObject(new PdfObjectId(3)).AsDictionary().Required().IsOfType(PdfName.Page).Should().BeTrue(); + } + + using var document = PdfDocument.Open(file.Bytes, ObjectGuard); + var section = document.Reader.Structure.Sections[^1]; + + section.CutByLimit.Should().BeTrue(); + section.Incomplete.Should().BeFalse(); + document.WasRepaired.Should().BeFalse(); } [Fact] From 8157bffe0daee0df313aade8f6de9c4e000d14a6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 05:18:59 +0000 Subject: [PATCH 134/146] fix: cut data a hidden length ends only where white space, or a comment no data holds, may lie The first filter under a hidden end decoded the data as taken and, where it found anything to report, again up to the run of white space and comments that ends it. A % the data starts with was taken for a comment and the whole data dropped; in ASCII85, where a % is a digit, a line starting with one was dropped and not marked cut; spaces after LZW codes spelled an end code taken for the data's own; the chain's Flate predictor ran on an ASCIIHex step; raw deflate, or a zlib checksum past bytes that look like a comment, lost its last bytes to the guard; and data of white space alone under ASCIIHex was marked cut. A comment now starts only past white space within the data, and ends no ASCII85 data. Flate data whose deflate body ends in the data as taken is whole, its repair reported, a checksum that disagrees only where it reaches into the run silenced. LZW, RunLength and ASCII data decode up to the run, their end markers counting only there, and the predictor runs on Flate and LZW steps alone. Data that is all white space under ASCII85 or ASCIIHex is whole. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../IO/Filters/PdfFilterPipeline.cs | 126 ++++++++++-------- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 98 ++++++++++++++ 2 files changed, 167 insertions(+), 57 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs index 16ab4b9d..961d06cf 100644 --- a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs +++ b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs @@ -157,19 +157,20 @@ private static ReadOnlyMemory DecodeChain( // Whether the data a filter is given ends where the file's may not: data the reader cut at one of its limits — // MaxStreamLength, or the window a guard stopped at, cut the stream's data, or a filter before it stopped at - // MaxDecodedStreamLength —, or data whose end the caller says a guard hides. Every filter after it is given such - // data, and a cut is reported once, by the guard that made it; data whose end is hidden is no cut. A filter drops a - // last group such data holds in part, and counts it against the bound all the same (Ascii85Filter, AsciiHexFilter), - // and against what the filters before the last share. + // MaxDecodedStreamLength —, or data whose end the caller says a guard hides, or what a first filter that ran out + // where a guard hides the data's end decoded. Every filter after it is given such data, and a cut is reported once, + // by the guard that made it. A filter drops a last group such data holds in part, and counts it against the bound + // all the same (Ascii85Filter, AsciiHexFilter), and against what the filters before the last share. var cut = stream.Data.CutByGuard || endHidden; // Data whose endstream is proven its own past white space and comments only a hidden length says are the data's or - // not (#224): the first filter decodes it as the search took it, and where it ends it there as its format says, - // with nothing to report, that is the data read whole. Otherwise it decodes it again up to where that run starts: - // what it meets before is the file's, its running out there says nothing of the file, and where its output may - // lack what the run holds — Flate that lost its tail, LZW or RunLength data without its end marker —, the output's - // end is the guard's, as of data a guard cut, for the filters after it and the reader of what the stream decodes - // to. White space and comments decode to nothing in ASCII85 or ASCIIHex data, whose output is whole. + // not (#224): the first filter decodes it up to where that run starts. What it meets before is the file's, its + // running out there says nothing of the file, and where its output may lack what the run holds — Flate that lost + // its tail, LZW or RunLength data without its end marker —, the output's end is the guard's, as of data a guard + // cut, for the filters after it and the reader of what the stream decodes to. Flate data whose deflate body ends + // in the data as the search took it is whole however the run reads, its checksum alone reaching into it. White + // space decodes to nothing in ASCII85 or ASCIIHex data, whose output is whole, and so does a comment in ASCIIHex, + // which no valid hexadecimal data holds; in ASCII85 a % is a digit, and no comment ends the data. var hidden = stream.Data.EndHidden && !cut; var endCut = false; @@ -196,26 +197,37 @@ private static ReadOnlyMemory DecodeChain( var maxLength = last ? bound : (int)Math.Clamp(shared, 0, bound); var given = data; var silenced = false; + var checksumHidden = false; if (hidden) { hidden = false; + silenced = true; + var run = HiddenRunStart(data.Span, comments: step.Name != PdfName.ASCII85Decode); - if (TryDecodeWhole(step, data, maxLength, out var whole)) + if (step.Name == PdfName.FlateDecode && FlateBodyEnds(data, maxLength, out var whole, out var clean)) { - data = ApplyPredictor(whole, step, diagnostics, stream.Data.Position, cut: false); - shared -= data.Length; - continue; + if (clean) + { + data = ApplyPredictor(whole, step, diagnostics, stream.Data.Position, cut: false); + shared -= data.Length; + continue; + } + + // A checksum that disagrees only where it reaches into the run says nothing of the file. + checksumHidden = run < data.Length && !ChecksumDisagrees(data[..run], maxLength); } - - silenced = true; - data = data[..HiddenRunStart(data.Span)]; - - // Data that is all white space and comments decodes to nothing, which may lack all the run holds. - if (data.IsEmpty) + else { - endCut = true; - break; + data = data[..run]; + + // Data that is all the run decodes to nothing, which may lack all the run holds, but in ASCII85 or + // ASCIIHex data. + if (data.IsEmpty) + { + endCut = step.Name != PdfName.ASCII85Decode && step.Name != PdfName.ASCIIHexDecode; + break; + } } } @@ -228,6 +240,7 @@ private static ReadOnlyMemory DecodeChain( guard, ref cut, silenced, + checksumHidden, ref limitReported, out var dropped, out var failed, @@ -262,12 +275,13 @@ private static ReadOnlyMemory DecodeChain( } /// - /// Gives where the run of white space and comments that ends starts: the bytes only a hidden - /// length says are the data's or not, as the gap between a length and its endstream may hold them (#174), a - /// comment there starting past white space, at the first % its line holds so. + /// Gives where the run of white space, and with of comments, that ends + /// starts: the bytes only a hidden length says are the data's or not, as the gap between a + /// length and its endstream may hold them (#174), a comment there starting past white space within the data, at + /// the first % its line holds so — one the data starts with is data. /// /// Each line is read twice at most: the run is found in time linear in the data. - private static int HiddenRunStart(ReadOnlySpan data) + private static int HiddenRunStart(ReadOnlySpan data, bool comments) { var start = data.Length; @@ -278,8 +292,13 @@ private static int HiddenRunStart(ReadOnlySpan data) start--; } + if (!comments) + { + return start; + } + // A comment runs to the end of its line, which the white space passed ends, or the data's end does: it starts at - // the first % its line holds past white space, or at the line's start, past the end-of-line before it. + // the first % its line holds past white space, the end-of-line before the line's start among it. var line = start; while (line > 0 && data[line - 1] is not ((byte)'\r' or (byte)'\n')) @@ -291,7 +310,7 @@ private static int HiddenRunStart(ReadOnlySpan data) for (var i = line; i < start && comment < 0; i++) { - if (data[i] == (byte)'%' && (i == line || PdfCharacters.IsWhitespace(data[i - 1]))) + if (data[i] == (byte)'%' && i > 0 && PdfCharacters.IsWhitespace(data[i - 1])) { comment = i; } @@ -307,36 +326,22 @@ private static int HiddenRunStart(ReadOnlySpan data) } /// - /// Decodes data whose end a guard hides as the search took it, and determines whether the filter ended it as its format - /// says, with nothing to report: its end marker or checksum met, no fault, the bound not reached. + /// Decodes Flate data whose end a guard hides as the search took it, and determines whether its deflate body ends in it + /// — its checksum whole, disagreeing or short —, the bound not reached: what it decoded to is then whole, and + /// says whether it is zlib data whose checksum agrees, with nothing to report. /// - private static bool TryDecodeWhole(in FilterStep step, ReadOnlyMemory data, int maxLength, out byte[] decoded) + private static bool FlateBodyEnds(ReadOnlyMemory data, int maxLength, out byte[] decoded, out bool clean) { - var name = step.Name; - bool limited; - - if (name == PdfName.FlateDecode) - { - return FlateFilter.TryDecode(data, out decoded, out var repaired, out var ending, out _, out limited, maxLength) && - !repaired && ending == FlateEnding.Whole && !limited; - } - - if (name == PdfName.LZWDecode) - { - decoded = LzwFilter.Decode(data.Span, step.Parameters.EarlyChange, out limited, out var undefinedCode, out var lzwEnding, maxLength); - return !limited && undefinedCode < 0 && lzwEnding == DataEnding.Marked; - } - - DataFaults faults; - decoded = name == PdfName.ASCII85Decode - ? Ascii85Filter.Decode(data.Span, out limited, out faults, maxLength, cut: false) - : name == PdfName.ASCIIHexDecode - ? AsciiHexFilter.Decode(data.Span, out limited, out faults, maxLength, cut: false) - : RunLengthFilter.Decode(data.Span, out limited, out faults, maxLength); - return !limited && faults.Ending == DataEnding.Marked && faults.Passed == 0 && faults.Overflowing == 0 && - faults.BrokenMarker == 0 && !faults.UnfinishedMarker && !faults.Introducer && !faults.LoneDigit; + var ended = FlateFilter.TryDecode(data, out decoded, out var repaired, out var ending, out _, out var limited, maxLength) && + !limited && ending is FlateEnding.Whole or FlateEnding.ChecksumMismatch or FlateEnding.ChecksumMissing; + clean = ended && !repaired && ending == FlateEnding.Whole; + return ended; } + /// Determines whether the checksum of Flate data disagrees with what it decodes to. + private static bool ChecksumDisagrees(ReadOnlyMemory data, int maxLength) => + FlateFilter.TryDecode(data, out _, out _, out var ending, out _, out _, maxLength) && ending == FlateEnding.ChecksumMismatch; + /// /// Determines whether a filter produces an image format that should be left encoded. /// @@ -359,13 +364,18 @@ public static bool IsImageFilter(PdfName name) => /// white space and comments only the hidden length says are the data's or not —: a filter that runs out there reports /// nothing, drops no group the end parts, and undoes a predictor's last row as far as it goes only where it ran out. /// + /// + /// Whether a Flate checksum that disagrees does so only where it reaches into what a hidden length says is the data's + /// or not, which says nothing of the file. + /// /// Whether the stream reported a filter that stopped at the bound already. /// How many bytes a last group the cut parted would decode to, which the filter did not write. /// Whether the filter could decode nothing, which ends the chain. /// /// Whether the filter ran out at the end of its data where its output may lack what more data would decode to: Flate - /// that lost its tail, LZW or RunLength data without its end marker. White space and comments, which decode to nothing in - /// ASCII85 and ASCIIHex data, are no more data for them. + /// that lost its tail, LZW or RunLength data without its end marker. White space, which decodes to nothing in ASCII85 + /// and ASCIIHex data, is no more data for them, nor a comment, which no ASCIIHex data holds, and which ends no ASCII85 + /// data. /// private static ReadOnlyMemory ApplyOne( in FilterStep step, @@ -376,6 +386,7 @@ private static ReadOnlyMemory ApplyOne( PdfLimitGuard guard, ref bool cut, bool silenced, + bool checksumHidden, ref bool limitReported, out int dropped, out bool failed, @@ -407,7 +418,8 @@ private static ReadOnlyMemory ApplyOne( // this one stopped decoding it, which the guard reported, and data whose end a guard hides ends where the guard // may: that it ran out says nothing of the file. A fault, or a checksum that disagrees, lies in bytes the reader // did read, and the whole data would meet it at the same point: it is the file's. - var endHidden = quiet && ending is FlateEnding.TailLost or FlateEnding.ChecksumMissing; + var endHidden = (quiet && ending is FlateEnding.TailLost or FlateEnding.ChecksumMissing) || + (checksumHidden && ending == FlateEnding.ChecksumMismatch); ReportEnding(endHidden ? FlateEnding.Whole : ending, decoded.Length, faultAt, data.Length, diagnostics, position); Reached(limited, name, bounds.Shared, diagnostics, position, guard, ref cut, ref limitReported); diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 7718095b..51aeb2f3 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -4,6 +4,7 @@ using AdCodicem.Pdf.Diagnostics; using AdCodicem.Pdf.Documents; using AdCodicem.Pdf.IO; +using AdCodicem.Pdf.IO.Filters; using AdCodicem.Pdf.IO.XRef; using AdCodicem.Pdf.Objects; using AdCodicem.Pdf.Validation; @@ -1341,6 +1342,103 @@ public void An_object_stream_whose_last_code_a_hidden_end_leaves_a_byte_short_en document.Diagnostics.Select(entry => entry.Code).Should().Equal(GuardCode(guard)); } + [Theory] + [InlineData("cut", "/ASCII85Decode", "%74#:-qIocCi:aF<,*OE;u", "\n", false)] + [InlineData("left out", "/ASCII85Decode", "%74#:\n%-qIocCi:aF<,*OE;u", "\n", false)] + [InlineData("cut", "/ASCII85Decode", "6<#'\\7PQ#?1*BP.+?)\n%u2_d'1<+I+\"87cURD]g__+B3(u78s", "\n", false)] + [InlineData("left out", "/ASCII85Decode", "<~6<#'\\7PQ#?1*BP.+?)\n%u2_d'1<+I+\"87cURD]g__+B3(u78s~>", "\n", false)] + [InlineData("cut", "/RunLengthDecode", "%xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "\n", true)] + [InlineData("cut", "/LZWDecode", "\u0080\u001C\u0040", " \n", true)] + [InlineData("left out", "/FlateDecode", "hello world \t", "\n", false)] + [InlineData("cut", "[/ASCIIHexDecode /FlateDecode] /DecodeParms << /Predictor 12 /Columns 4 >>", "{predicted}>", "\n", false)] + [InlineData("cut", "[/LZWDecode /ASCIIHexDecode]", "{lzw-hex}", "\n", true)] + [InlineData("left out", "[/RunLengthDecode /ASCIIHexDecode]", "\u0006717171\n", "", true)] + [InlineData("cut", "/ASCIIHexDecode", " ", "\n", false)] + public void Data_a_hidden_length_ends_decodes_as_read_whole_but_for_what_the_run_after_it_may_hold( + string guard, string filter, string data, string tail, bool cut) + { + // Content stream 4's data, then white space before an endstream the next object proves its own, under a /Length a + // guard cut or left out of the index (#224). Only the run of white space, and of comments past white space where no + // data holds a %, is what the hidden length may leave out. A % the data starts with was taken for a comment and the + // whole data dropped; a % at a line's start, a digit in ASCII85, the rest of the line; spaces after LZW codes read + // as their end code; the chain's Flate predictor run on an ASCIIHex step; data that is only white space under + // ASCIIHex taken for cut. What the first filter decodes is the whole read's, and only where it may lack what the run + // holds — LZW or RunLength data without its end marker — is its end the guard's, for the filters after it too. + var bytes = data switch + { + "{predicted}>" => Encoding.ASCII.GetBytes(Convert.ToHexString(Zlib([2, 1, 2, 3, 4, 2, 0, 0, 0, 0])) + ">"), + "{lzw-hex}" => FilterDamageTests.NineBitCodes([256, '7', '1', '7', '1', '7', '1', '7', '1', '7', '1', '7', '1', '7', '1', 13]), + _ => Encoding.Latin1.GetBytes(data), + }; + var file = HiddenLengthFile(guard, "/Filter " + filter, bytes, tail); + byte[] whole; + + using (var document = PdfDocument.Open(file)) + { + whole = document.GetObject(new PdfObjectId(4)).AsStream().Required().Decode().ToArray(); + } + + using var guarded = PdfDocument.Open(file, HiddenLengthOptions(guard)); + var stream = guarded.GetObject(new PdfObjectId(4)).AsStream().Required(); + var decoded = PdfFilterPipeline.Decode(stream, null, out _, out var cutByGuard).ToArray(); + + stream.Data.EndHidden.Should().BeTrue(); + decoded.Should().Equal(whole); + cutByGuard.Should().Be(cut); + guarded.Diagnostics.Where(entry => entry.Code == PdfDiagnosticCodes.FilterFailed && entry.Message.Contains("end", StringComparison.Ordinal)) + .Should().BeEmpty(); + } + + [Theory] + [InlineData("cut", 12)] + [InlineData("left out", 2)] + public void Flate_data_a_hidden_length_ends_where_its_deflate_body_does_decodes_as_read_whole(string guard, int predictor) + { + // Content stream 4's Flate data, under a /Length a guard cut or left out of the index, its endstream proven its own: + // raw deflate whose last byte is a NUL, zlib whose checksum is wrong past bytes that look like a comment after white + // space, zlib whose predicted rows end it whole. The deflate body ends in the data as the search took it: what it + // decodes to is whole, the repair raw deflate asks for reported as read whole. It was decoded again up to the NUL, or + // to the % the checksum's bytes hold, its last bytes lost and its end taken for the guard's (#224). + var raw = new MemoryStream(); + + using (var deflate = new DeflateStream(raw, CompressionLevel.Optimal, leaveOpen: true)) + { + deflate.Write("BT /F1 12 Tf 72 712 Td (Hello, world) Tj ET"u8); + } + + byte[] stored = [0x78, 0x01, 0x01, 15, 0, unchecked((byte)~15), 0xFF, .. "BT (a %b) Tj ET"u8, 0, 0, 0, 0]; + byte[] rows = predictor == 2 ? [1, 2, 3, 4, 1, 1, 1, 1] : [2, 1, 2, 3, 4, 2, 0, 0, 0, 0]; + (string Entries, byte[] Data)[] cases = + [ + ("/Filter /FlateDecode", raw.ToArray()), + ("/Filter /FlateDecode", stored), + (Invariant($"/Filter /FlateDecode /DecodeParms << /Predictor {predictor} /Columns 4 >>"), Zlib(rows)), + ]; + + raw.ToArray()[^1].Should().Be(0); + + foreach (var (entries, data) in cases) + { + var file = HiddenLengthFile(guard, entries, data, "\n"); + byte[] whole; + string[] faults; + + using (var document = PdfDocument.Open(file)) + { + whole = document.GetObject(new PdfObjectId(4)).AsStream().Required().Decode().ToArray(); + faults = [.. document.Diagnostics.Where(entry => entry.Code == PdfDiagnosticCodes.FilterFailed).Select(entry => entry.Message)]; + } + + using var guarded = PdfDocument.Open(file, HiddenLengthOptions(guard)); + var stream = guarded.GetObject(new PdfObjectId(4)).AsStream().Required(); + + PdfFilterPipeline.Decode(stream, null, out _, out var cutByGuard).ToArray().Should().Equal(whole); + cutByGuard.Should().BeFalse(); + guarded.Diagnostics.Where(entry => entry.Code == PdfDiagnosticCodes.FilterFailed).Select(entry => entry.Message) + .Should().Equal(faults); + } + } + /// /// A file whose content stream 4 holds and before its endstream, which /// object 5 after it proves its own, its /Length object 20, which a MaxObjectLength of 1,024 cuts, or, left From 3291bf7f8a2c22110386a9ba3683eff060e7808e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 05:24:42 +0000 Subject: [PATCH 135/146] fix: find a deferred length's row in the predicted row a spelled endstream parts A cross-reference stream whose /Length only its own rows place is taken to the first endstream, and its rows may spell one. The section's own row for that /Length is read from the rows held, but where the spelled endstream parts the predicted row that holds it, the data, decoded as data whose end the file gives, dropped that row: the length was not found, the section was charged with the rows it lacked, and a valid file's index was rebuilt. The data is now decoded once more knowing its end hidden, its parted row undone as far as it goes, and the row read from there; what that decoding reports is dropped with it. Refs #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 28 ++++++++-- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 52 +++++++++++++++++++ .../DocumentReaderTests.cs | 18 +++++++ 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 9da562d2..32874673 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -4137,7 +4137,25 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect // is asked too: a newer section may place the object, which a guard cut. if (!retaken && !endHidden && deferred is { } unsettled && declared > held && decodedWhole) { - if (RetakeDeferred(xrefStream, unsettled, data, fieldWidths, row, held, index is null ? [0, declared] : subsections, out lengthCut) is { } whole) + long[] ranges = index is null ? [0, declared] : subsections; + var whole = RetakeDeferred(xrefStream, unsettled, data, fieldWidths, row, held, ranges, out lengthCut, out var past); + + // The row for the /Length may lie in a predicted row the spelled endstream parts, which the data decoded as + // data whose end the file gives dropped: the data decoded knowing its end hidden undoes it as far as it goes, + // and holds the rows before the endstream whole. What that decoding reports is dropped with it. + if (whole is null && past && row > 0) + { + var again = _diagnostics.GetMarkWithDecodingReports(); + var undone = Filters.PdfFilterPipeline.Decode(xrefStream, _diagnostics, endHidden: true, reachedBefore: false, out var leftAgain, out _); + _diagnostics.RollBack(again); + + if (!leftAgain && undone.Length / row > held) + { + whole = RetakeDeferred(xrefStream, unsettled, undone.Span, fieldWidths, row, undone.Length / row, ranges, out lengthCut, out _); + } + } + + if (whole is not null) { _diagnostics.RollBack(decoding); xrefStream = whole; @@ -4309,7 +4327,8 @@ private bool EndsAtOwnEndStream(PdfStream stream, ReadOnlySpan data, ReadO /// gives, which the chain could not read and took the data up to the first endstream for, when the rows /// holds show that endstream was in the rows (#220): the section's own row for the object /// the /Length names, among those held, places it, and the length it gives runs past the data taken, to an - /// endstream that confirms it. Null otherwise: the length is checked once the chain is read. + /// endstream that confirms it. Null otherwise: the length is checked once the chain is read — and the row lies + /// past those held where past says so. /// /// /// The section's row for the object is the first that numbers it with a type the reading of its rows takes: one of a @@ -4332,9 +4351,11 @@ private bool EndsAtOwnEndStream(PdfStream stream, ReadOnlySpan data, ReadO int row, long held, long[] subsections, - out bool lengthCut) + out bool lengthCut, + out bool past) { lengthCut = false; + past = false; var number = deferred.Length.Number; if (_xref.TryGet(number, out _) || _refusedNumbers?.Contains(number) == true) @@ -4361,6 +4382,7 @@ private bool EndsAtOwnEndStream(PdfStream stream, ReadOnlySpan data, ReadO if (at >= held) { + past = true; return null; } diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index af708a09..2455ed6a 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -852,6 +852,58 @@ byte[] Data(long five, long stream) return file.ToArray(); } + /// + /// A file of one revision: the catalog, the page tree, the page, object 5, which holds the length of cross-reference + /// stream 7's data, and stream 7, Flate in a stored block with a PNG predictor of columns, + /// whose /Length names object 5, which only its own rows place: the free row 0's, the catalog's, the page tree's + /// and the page's, then object 5's, two of reserved types that spell endstream after an end-of-line, and its own + /// (/Index [0 4 5 1 19 2 7 1]). With 28 columns, the spelled endstream parts the second predicted row + /// right after object 5's row. The file is valid. + /// + public static byte[] DeferredLengthRowPartedBySpelledEndStream(int columns) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + + // The data's length does not depend on the offsets its rows hold. + byte[] Data(long five, long stream) + { + var rows = new List(Encode( + [(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), (1, five, 0)], + Rows.Raw)); + rows.AddRange("\nendstream\0\0\0\0"u8.ToArray()); + rows.AddRange(Encode([(1, stream, 0)], Rows.Raw)); + + while (rows.Count % columns != 0) + { + rows.Add(0); + } + + var predicted = new List(); + + for (var i = 0; i < rows.Count; i += columns) + { + predicted.Add(0); + predicted.AddRange(rows.GetRange(i, columns)); + } + + return Compress([.. predicted]); + } + + file.Object(5, Data(0, 0).Length.ToString(CultureInfo.InvariantCulture)); + var at = file.Position; + file.Stream( + 7, + string.Create( + CultureInfo.InvariantCulture, + $"/Type /XRef /Size 21 /W [1 4 2] /Index [0 4 5 1 19 2 7 1] /Filter /FlateDecode /DecodeParms << /Predictor 12 /Columns {columns} >> /Root 1 0 R /Length 5 0 R"), + Data(file.OffsetOf(5), at)); + file.StartXRef(at); + return file.ToArray(); + } + /// /// A file of one cross-reference stream, object 7, whose /Length names object 5, which only the stream's own row /// places, before the rows of objects 1 and 2; two rows of reserved types spell endstream after an end-of-line diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index bce88c64..6aeb3dae 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -3125,4 +3125,22 @@ protected override void Dispose(bool disposing) base.Dispose(disposing); } } + + [Theory] + [InlineData(28)] + [InlineData(21)] + public void Reads_a_cross_reference_stream_whose_length_row_a_spelled_endstream_parts_the_predicted_row_of(int columns) + { + // Stream 7's /Length names object 5, which only its own row places, and its rows spell an endstream right after that + // row, which parts the second predicted row: the data up to it decodes, as data whose end the file gives, to the first + // row alone, the parted one dropped, and the row for the /Length was not found. The section was charged with the + // rows it lacked and the index rebuilt (#220). + var file = ChainFiles.DeferredLengthRowPartedBySpelledEndStream(columns); + + using var document = PdfDocument.Open(file); + + document.Diagnostics.Should().BeEmpty(); + document.WasRepaired.Should().BeFalse(); + document.GetObject(new PdfObjectId(3)).AsDictionary().Required().IsOfType(PdfName.Page).Should().BeTrue(); + } } From 59c68ee2de3b1b035b4f64e1d0dfb8e552e31975 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 05:27:10 +0000 Subject: [PATCH 136/146] docs: say how a hidden end decodes and that a section lacking rows beside a cut reads whole The architecture, the references, the concepts, the threat model and the XML docs said data a hidden length ends decodes as read whole where its first filter ends it with nothing to report. They now say what the code does: the first filter decodes up to the run of white space, and of comments past white space within the data but in ASCII85, and Flate data whose deflate body ends in the data as taken is whole. A cross-reference stream lacking rows beside a value a guard cut is said to be read as read whole, its filters written direct, rather than unreadable. A deferred length's row in a predicted row a spelled endstream parts, the search costs past the window, and the comments the concept pages left out are said as well. Refs #220, #224, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- ...-is-readable-and-the-readers-guards-are.md | 2 +- docs/architecture.md | 12 ++++++---- docs/threat-model.md | 20 ++++++++-------- docs/website/docs/concepts/diagnostics.md | 4 ++-- docs/website/docs/concepts/reader-limits.md | 11 +++++---- docs/website/docs/concepts/validation.md | 5 ++-- docs/website/docs/reference/diagnostics.md | 23 ++++++++++--------- docs/website/docs/reference/reader-limits.md | 9 ++++---- .../docs/reference/validation-rules.md | 2 +- .../IO/Filters/PdfFilterPipeline.cs | 8 +++---- src/AdCodicem.Pdf/Objects/PdfStreamData.cs | 6 ++--- 11 files changed, 54 insertions(+), 48 deletions(-) diff --git a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md index 70b9d449..0a7ba9f2 100644 --- a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md +++ b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md @@ -126,7 +126,7 @@ on 2026-10-24) —, stops the search, the data up to it then the guard's; a whole reference the cut left leads on, as read whole; a chain of filters ends there, its data left encoded; and a cross-reference section whose values lead to an object `MaxObjectLength` cut ends the -chain, nothing rebuilt — unless what the values read, and data whose length the file gives, say makes it unreadable whatever the cut hides, the file's fault +chain, nothing rebuilt — unless what the values read say makes it unreadable whatever the cut hides, or data whose length the file gives, its filters written direct, lacks the rows they declare whatever it hides — the file's fault, such a section read as read whole, lacking rows (amended on 2026-10-24) —, the older sections and their trailers the guard's to hide, and the guard reached once the chain is read. So does a cross-reference stream's `/Length` the chain could not read that leads to such an object, wherever only that length could say how far the data runs: no `endstream` comes before the next object or the end of the file, or diff --git a/docs/architecture.md b/docs/architecture.md index a5e94a02..cc3a7d8e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -142,8 +142,9 @@ separate types, separate severity scales, and vocabularies that never share a co through, as a load reads it; its chain of filters ends there, with the data left encoded; and the chain of cross-reference sections ends at that section, with nothing rebuilt and the older sections and their trailers the guard's to hide, wherever near its named offset the section is found — - unless what the values read, and data whose length the file gives, say makes it unreadable whatever the cut - hides, which is the file's fault. An + unless what the values read say makes it unreadable whatever the cut hides, or data whose length the file + gives, its filters written direct, lacks the rows they declare whatever it hides — the file's fault, such a section + read as read whole, lacking rows, its trailer merged and the chain going on. An object `MaxObjectLength` cut there is loaded once the chain is read, which reaches the guard — thrown at opening under `ThrowOnLimit` —, for a value the chain keeps: not for a place it dropped as no section. A cross-reference stream's `/Length` the chain could not read, which leads to such an object, ends the chain so wherever only it @@ -153,9 +154,10 @@ separate types, separate severity scales, and vocabularies that never share a co proves it the stream's own, as in item 4, which leaves the rows the data lacks to the file. Such data, whose end a guard hides, is decoded again as data a guard cut, a group its end parts dropped and what a filter says of the end silenced; a decoding guard the first decoding reached stands, and no more rows are held than it read. - Data an `endstream` proven its own ends past white space or comments (item 4) decodes as read whole where its - first filter ends it as its format says; otherwise up to where that run starts, the filter's running out there - silenced, and where its output may lack what the run holds, its end the guard's, as cut. + Data an `endstream` proven its own ends past white space or comments (item 4) decodes up to where that run + starts — a comment past white space within the data, but in ASCII85 data, where a `%` is a digit —, the first + filter's running out there silenced, and where its output may lack what the run holds, its end the guard's, as cut; + Flate data whose deflate body ends in the data as taken is whole, its checksum alone reaching into the run. ### 3.2 The write path diff --git a/docs/threat-model.md b/docs/threat-model.md index 36abc77e..d48ee2a7 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -283,8 +283,8 @@ Known gaps: | A classic table of any length | Its window grows ×4 from 64 KB up to `MaxXRefSectionLength`, and the rows read are kept. A table the guard cuts, or whose rows run past a full index, keeps its trailer, found past the rows left unread by a search a megabyte at a time (`ReadTrailerPastRows`): comments are skipped, and anything else no row holds is the section's fault, reported, the search going on to the `trailer` keyword, so that its `/Encrypt` still refuses the document. The search starts at the start of the line the cut fell in, outside any comment and past the table's own keyword; it reads each megabyte once, every byte of it looked at before the next is read, however many comments or stray bytes it holds, and is not made for a place a section may have been moved to, a guess. The chain ends there, since an older section's rows would stand for the objects the unread rows define. What those rows place reads as null until the guard is raised | `ReaderLimitsTests.A_table_whose_keywords_the_bound_cuts_reports_the_bound`, `A_table_its_length_bound_cuts_keeps_its_trailer_and_the_encryption_it_names`, `A_table_its_length_bound_cuts_ends_the_chain_though_its_trailer_names_an_older_section`, `What_ends_the_rows_a_length_bound_cut_is_judged_as_it_is_where_none_did`, `A_table_a_length_bound_cuts_that_runs_to_the_end_of_the_file_has_no_trailer`, `A_table_its_length_bound_cuts_keeps_the_encryption_its_trailer_names_past_what_no_row_holds`, `A_table_cut_before_its_keyword_ends_keeps_its_trailer`, `Rows_past_a_full_index_are_not_read_and_the_trailer_past_them_is`, `A_cut_in_a_comment_among_the_rows_searches_for_the_trailer_from_the_comment_s_start`, `The_trailer_past_rows_left_unread_is_found_wherever_the_search_s_window_ends`, `A_trailer_past_rows_left_unread_that_its_own_bound_cuts_keeps_what_precedes_the_cut`; `DocumentReaderTests.The_search_for_the_trailer_past_rows_left_unread_reads_the_file_once`; `CrossReferenceChainTests.Reads_an_object_a_length_bound_cut_out_of_its_table_as_null_until_the_guard_is_lifted`, `A_section_found_near_where_prev_names_it_is_not_searched_past_the_rows_a_guard_left_unread`; `WindowEdgeTests.A_classic_table_reads_whole_wherever_its_window_edge_falls_near_its_trailer` | No: seeds stay under 120 KB | | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). A hybrid file's stream's own `/Prev`, which the chain does not follow, going on through the table's, is not read at all, nor are the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place: neither names a section missing nor ends the chain for a guard, and the rule on trailer values judges them as written, a reference they hold not followed ([#263]). An object the guard cut after a whole reference it holds hides nothing: the chain reads through it, as a load does, and loads it once the chain is read. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s`, `A_table_an_xrefstm_names_in_a_stream_s_place_has_its_own_links_left_unread`, `A_prev_written_as_a_reference_to_a_real_is_judged_through_it_only_where_the_reader_follows_it` | Yes | -| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); so is one that declared itself a section before a guard cut it — a table's `xref` keyword, `/Type /XRef`. A place that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is a guess dropped, the next place tried; where the chain names a section, a guard that cut the dictionary of what declares another type cut no section — a CIDFont's, which read whole no stream data follows —, and relocation is tried, while one that cut a value of a stream there leaves it the section the guard cut, whatever type it gives, as read whole it is the section. What makes a section unreadable whatever a value a guard cut holds is the file's fault, as read whole, judged before the cut is taken for the guard's (`PdfFileReader.FaultBesideCut`): a value past the cut that no section read so far places, which the sections a whole read reads before make the file's to lack; a `/W` that gives no three widths, or none past 0; rows numbered wrongly — an `/Index` that holds no pairs, whatever its elements, a subsection out of range, or wrong by the element read whatever the cut one holds, a `/Size` with no `/Index` that gives no count —, as far as the values read, written direct or through references, say: a value a guard cut says nothing, nor does what depends on it, the `/Size` an `/Index` so cut may stand beside; and data whose length the file gives, its filters written direct, decoded whole, that holds fewer rows than the counts read declare however few bytes the widths read leave a row, which lacks them whatever the cut hides: the section is there, not looked for nearby, and the index rebuilt | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_cross_reference_stream_whose_deferred_length_a_guard_cut_is_found_where_it_was_moved`, `A_cross_reference_stream_a_guard_cut_the_dictionary_of_after_its_type_is_the_section_where_it_was_moved`, `A_classic_table_a_guard_cut_the_rows_of_is_the_section_where_it_was_moved`, `A_stream_with_no_numbering_whose_width_a_guard_cut_is_no_section_the_guard_cut`, `A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild`, `What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_guard_cut_holds_is_the_file_s`, `A_place_a_section_may_have_moved_to_that_a_value_past_a_guard_s_cut_makes_unreadable_is_dropped`, `A_stream_whose_value_a_guard_cut_where_the_chain_names_a_section_is_the_section_the_guard_cut_whatever_its_type`, `A_cross_reference_stream_that_misspells_its_type_and_whose_value_a_guard_cut_is_the_section_the_guard_cut`, `A_section_a_value_s_cut_keeps_from_being_read_is_the_guard_s_where_nothing_read_says_it_is_wrong`, `A_section_whose_data_holds_fewer_rows_than_it_declares_whatever_a_value_a_guard_cut_holds_is_the_file_s_to_lack` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, may stay within the bound — the group counted against what the filters before the last share as well, and no more rows held than the first decoding read —: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_decoding_guard_deferred_data_reached_is_reported_once_whatever_room_the_diagnostics_have`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `Data_whose_end_a_guard_hides_leaves_the_filters_after_a_dropped_group_the_room_its_direct_twin_s_have`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file`, `A_cross_reference_stream_the_chain_read_proves_the_endstream_of_the_older_one_before_it` | No | +| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); so is one that declared itself a section before a guard cut it — a table's `xref` keyword, `/Type /XRef`. A place that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is a guess dropped, the next place tried; where the chain names a section, a guard that cut the dictionary of what declares another type cut no section — a CIDFont's, which read whole no stream data follows —, and relocation is tried, while one that cut a value of a stream there leaves it the section the guard cut, whatever type it gives, as read whole it is the section. What makes a section unreadable whatever a value a guard cut holds is the file's fault, as read whole, judged before the cut is taken for the guard's (`PdfFileReader.FaultBesideCut`): a value past the cut that no section read so far places, which the sections a whole read reads before make the file's to lack; a `/W` that gives no three widths, or none past 0; rows numbered wrongly — an `/Index` that holds no pairs, whatever its elements, a subsection out of range, or wrong by the element read whatever the cut one holds, a `/Size` with no `/Index` that gives no count —, as far as the values read, written direct or through references, say: a value a guard cut says nothing, nor does what depends on it, the `/Size` an `/Index` so cut may stand beside; and data whose length the file gives, its filters written direct, decoded whole, that holds fewer rows than the counts read declare however few bytes the widths read leave a row — a width past what an int counts taken as such —, which lacks them whatever the cut hides: the section is read as read whole, lacking rows, its trailer merged, the chain going on to its `/Prev` and the index rebuilt, where it was moved to too, though the rows it holds are not read | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_cross_reference_stream_whose_deferred_length_a_guard_cut_is_found_where_it_was_moved`, `A_cross_reference_stream_a_guard_cut_the_dictionary_of_after_its_type_is_the_section_where_it_was_moved`, `A_classic_table_a_guard_cut_the_rows_of_is_the_section_where_it_was_moved`, `A_stream_with_no_numbering_whose_width_a_guard_cut_is_no_section_the_guard_cut`, `A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild`, `What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_guard_cut_holds_is_the_file_s`, `A_place_a_section_may_have_moved_to_that_a_value_past_a_guard_s_cut_makes_unreadable_is_dropped`, `A_stream_whose_value_a_guard_cut_where_the_chain_names_a_section_is_the_section_the_guard_cut_whatever_its_type`, `A_cross_reference_stream_that_misspells_its_type_and_whose_value_a_guard_cut_is_the_section_the_guard_cut`, `A_section_a_value_s_cut_keeps_from_being_read_is_the_guard_s_where_nothing_read_says_it_is_wrong`, `A_section_whose_data_holds_fewer_rows_than_it_declares_whatever_a_value_a_guard_cut_holds_is_the_file_s_to_lack`, `Data_a_length_the_chain_cannot_read_yet_leaves_to_a_spelled_endstream_says_nothing_of_the_rows_it_holds` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken, in a predicted row the spelled `endstream` parts too, the data decoded once more knowing its end hidden, that row undone as far as it goes and the reports dropped: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, may stay within the bound — the group counted against what the filters before the last share as well, and no more rows held than the first decoding read —: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_decoding_guard_deferred_data_reached_is_reported_once_whatever_room_the_diagnostics_have`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `Data_whose_end_a_guard_hides_leaves_the_filters_after_a_dropped_group_the_room_its_direct_twin_s_have`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file`, `A_cross_reference_stream_the_chain_read_proves_the_endstream_of_the_older_one_before_it`, `DocumentReaderTests.Reads_a_cross_reference_stream_whose_length_row_a_spelled_endstream_parts_the_predicted_row_of` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, written direct — an integer, or a real with no fractional part; one written as a reference is not read through for it, and bounds nothing —, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | @@ -357,8 +357,8 @@ Known gaps: | A declared length past the file | It is checked against the file before the file is asked, and confirms nothing: the stream goes to the search, whose `endstream` settles its length, and with none its data ends as the rows below say. The length a stream is given is clamped to what the file holds | `HostileInputTests.Ignores_a_stream_length_larger_than_the_file`, `A_length_the_file_cannot_hold_is_settled_by_the_endstream_found_without_reaching_a_guard`, `Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_length_the_file_cannot_hold_with_no_endstream_takes_the_rest_of_the_file_without_reaching_a_guard`; `WindowEdgeTests.A_stream_whose_length_the_file_cannot_hold_is_read_to_its_endstream` | Yes | | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | -| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space and comments the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`). Where the first filter ends the data as its format says, with nothing to report, it decodes as read whole; otherwise the filter decodes it again up to where that run starts, in time linear in the data: what it meets before is the file's — a checksum that disagrees with the bytes before it —, its running out there — a Flate checksum short, an end marker missing — says nothing of the file, and where its output may lack what the run holds — Flate that lost its tail, LZW or RunLength data without its end marker — the output's end is the guard's, cut, for the predictor, which undoes its last row as far as it goes, the filters after it and an object stream's last member; white space and comments decode to nothing in ASCII85 or ASCIIHex data, whose output is whole, and the filters after a first that ended whole decode as read whole, their faults the file's. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `A_cross_reference_stream_named_before_white_space_proves_the_endstream_before_it_however_much`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `Data_that_ends_in_white_space_right_before_a_proven_endstream_ends_where_the_guard_hides`, `A_checksum_short_of_a_byte_white_space_before_a_proven_endstream_completes_is_no_fault_of_the_file`, `A_checksum_that_disagrees_before_the_white_space_a_hidden_length_leaves_is_the_file_s`, `A_filter_after_one_that_ends_where_its_format_says_decodes_as_read_whole_whatever_hides_the_end`, `Predicted_rows_a_hidden_end_leaves_a_byte_short_are_undone_as_far_as_they_go`, `Lzw_rows_a_hidden_end_leaves_a_byte_short_are_undone_as_far_as_they_go`, `Lzw_data_that_lacks_its_end_code_before_a_proven_endstream_past_an_end_of_line_is_no_fault_of_the_file`, `White_space_and_comments_a_hidden_length_leaves_are_not_decoded_as_data_that_lacks_its_end_marker`, `An_object_stream_whose_last_code_a_hidden_end_leaves_a_byte_short_ends_where_the_guard_hides`, `An_ascii85_object_stream_that_lacks_its_end_marker_before_a_proven_endstream_decodes_as_read_whole`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | -| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file at most, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, and the white space it crosses once more at most — the run before an `endobj` taken from the read-on that read it, a run a row names before the next header read once from the row —, about three times the file in all; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `DocumentReaderTests.Deferred_lengths_followed_by_long_white_space_have_it_read_on_once_however_their_data_ends`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_proofs_of_streams_white_space_follows_read_less_than_three_times_the_file`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | +| A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space and comments the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`). The first filter decodes the data up to where that run starts, found in time linear in the data — a comment past white space within the data, never at its first byte, and none in ASCII85 data, where a `%` is a digit —: what it meets before is the file's, its end marker counting only there, its running out there — a Flate checksum short, an end marker missing — says nothing of the file, and where its output may lack what the run holds — Flate that lost its tail, LZW or RunLength data without its end marker — the output's end is the guard's, cut, for the predictor, which runs on Flate and LZW steps only and undoes the last row as far as it goes, the filters after it and an object stream's last member. Flate data whose deflate body ends in the data as taken decodes whole, its repair reported and a checksum that disagrees only where it reaches into the run silenced; white space decodes to nothing in ASCII85 or ASCIIHex data, whose output is whole, and so does a comment in ASCIIHex, which no valid hexadecimal data holds; the filters after a first whose output is whole decode as read whole, their faults the file's. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `A_cross_reference_stream_named_before_white_space_proves_the_endstream_before_it_however_much`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `Data_that_ends_in_white_space_right_before_a_proven_endstream_ends_where_the_guard_hides`, `A_checksum_short_of_a_byte_white_space_before_a_proven_endstream_completes_is_no_fault_of_the_file`, `A_checksum_that_disagrees_before_the_white_space_a_hidden_length_leaves_is_the_file_s`, `A_filter_after_one_that_ends_where_its_format_says_decodes_as_read_whole_whatever_hides_the_end`, `Predicted_rows_a_hidden_end_leaves_a_byte_short_are_undone_as_far_as_they_go`, `Lzw_rows_a_hidden_end_leaves_a_byte_short_are_undone_as_far_as_they_go`, `Lzw_data_that_lacks_its_end_code_before_a_proven_endstream_past_an_end_of_line_is_no_fault_of_the_file`, `White_space_and_comments_a_hidden_length_leaves_are_not_decoded_as_data_that_lacks_its_end_marker`, `An_object_stream_whose_last_code_a_hidden_end_leaves_a_byte_short_ends_where_the_guard_hides`, `Data_a_hidden_length_ends_decodes_as_read_whole_but_for_what_the_run_after_it_may_hold`, `Flate_data_a_hidden_length_ends_where_its_deflate_body_does_decodes_as_read_whole`, `An_ascii85_object_stream_that_lacks_its_end_marker_before_a_proven_endstream_decodes_as_read_whole`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | +| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file, three times over the white space a cross-reference stream's search takes as data that its length, once read, is read on through, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, and the white space it crosses once more at most — the run before an `endobj`, or before what follows the keyword where no `endobj` does, taken from the read-on that read it, a run a row names before the next header read once from the row —, about three times the file in all; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `DocumentReaderTests.Deferred_lengths_followed_by_long_white_space_have_it_read_on_once_however_their_data_ends`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_proofs_of_streams_white_space_follows_read_less_than_three_times_the_file`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | | Encoded data of any length: a `/Length` of a few digits claiming gigabytes, one past what an array holds, which would exhaust memory at each read, or data decoding hands back as it is, which `MaxDecodedStreamLength` does not bound | `MaxStreamLength` (*The guards*): what is read of one stream's data at once stops at it, what lies past it is cut by a guard and reported as `limit.stream-length` — once for the stream in its document's diagnostics, and with each decoding that meets the cut in diagnostics a caller passes to `Decode`, as `limit.decoded-stream` is —, and the filters decode what was read as data the reader cut ([#170]) | `ReaderLimitsTests.A_stream_cut_at_MaxStreamLength_decodes_what_lies_within_it_as_data_the_reader_cut_and_is_reported_once`, `Data_decoding_hands_back_as_it_is_is_bounded_by_MaxStreamLength_and_not_by_MaxDecodedStreamLength`, `A_length_past_what_an_array_holds_ends_at_the_guard_and_never_in_an_exhausted_memory`, `Decoding_into_diagnostics_of_one_s_own_reports_the_cut_max_stream_length_made_each_time` | No | @@ -384,12 +384,12 @@ Known gaps: there: only the hidden length could tell, and a valid file's last stream before its table is told so ([#224]). - Accepted, with no issue: under a `/Length` a guard cut or left out of the index, data an `endstream` proven its own ends past white space or comments — the end-of-line the search leaves out before the keyword, above all — decodes - with its end the guard's where its first filter does not end it as its format says: that filter's running out — a - Flate stream that lost its tail or its checksum, ASCII85, ASCIIHex, RunLength or LZW data without its end marker — is - not reported there, though a whole read reports it, nor what the filters after it meet of an end the guard's; a - checksum whose bytes reach into the run says nothing; and where the data's last bytes, read whole, look like white - space or a comment after white space, what a filter that ran out decodes from them is cut, the guard's. Only the hidden - length says whether that run is the data's ([#224]). + up to where that run starts: its first filter's running out there — a Flate stream that lost its tail or its + checksum, ASCII85, ASCIIHex, RunLength or LZW data without its end marker — is not reported, though a whole read + reports it, nor what the filters after it meet of an end the guard's; a checksum whose bytes reach into the run says + nothing; and where the data's last bytes, read whole, look like white space or a comment after white space, what a + binary filter that ran out decodes from them is cut, the guard's. Only the hidden length says whether that run is the + data's ([#224]). - Accepted, with no issue: where the chain names a section, an object that declares another type and gives a `/W` before `MaxTrailerLength` cuts its dictionary is taken for no section, as a CIDFont's dictionary is read whole, no stream data after it, and the section is looked for nearby; a stream of that type, whose stream data the cut hides, diff --git a/docs/website/docs/concepts/diagnostics.md b/docs/website/docs/concepts/diagnostics.md index ac379911..1fefdc0d 100644 --- a/docs/website/docs/concepts/diagnostics.md +++ b/docs/website/docs/concepts/diagnostics.md @@ -77,8 +77,8 @@ nothing is said of the file. Only what follows the object's end — its `endobj` can prove an `endstream` the stream's own: the next object the index places, or a cross-reference stream the chain read, or the end of the file, which data that goes on is never followed by, or the `xref` or `startxref` keyword, which the data may spell as well — data that spells `endstream`, then one of them, and goes on is taken to end there, which only the hidden length could tell. The end is -then the file's, and so is an `endobj` the object lacks — but for white space before the keyword, an end-of-line -above all, which only the hidden length says is the data's or not: its end is decoded as one the guard hides, and a +then the file's, and so is an `endobj` the object lacks — but for white space or comments before the keyword, an +end-of-line above all, which only the hidden length says are the data's or not: its end is decoded as one the guard hides, and a filter that runs out there, a Flate checksum short of a carriage return, says nothing of the file. A whole reference the cut left, `6 0 R` before the window's end, is no length it hid: it leads on to the object it names, as read whole. diff --git a/docs/website/docs/concepts/reader-limits.md b/docs/website/docs/concepts/reader-limits.md index 4ed114f6..f64c9620 100644 --- a/docs/website/docs/concepts/reader-limits.md +++ b/docs/website/docs/concepts/reader-limits.md @@ -59,13 +59,14 @@ too. An `endstream` is taken for the stream's own only where what follows its ob index places, a cross-reference stream the chain read, or the end of the file, which data that goes on is never followed by, or a cross-reference section's keyword, `xref` or `startxref`, which the data may spell as well: data that spells `endstream`, then one of them, and goes on is taken to end there, which only the hidden length could tell. The end is -then the file's, though white space before the keyword, which only the length says is the data's or not, leaves where it -ends to the guard as the data decodes: a filter that runs out there says nothing of the file. Decoding stops at a filter so named, or one whose parameters are, and leaves +then the file's, though white space or comments before the keyword, which only the length says are the data's or not, +leave where it ends to the guard as the data decodes: a filter that runs out there says nothing of the file. Decoding stops at a filter so named, or one whose parameters are, and leaves the rest of the data encoded, as past the most filters the reader runs; and a `/Prev` or an `/XRefStm` the reader follows that is so named, or a cross-reference stream whose values lead to an object a guard cut, ends the chain as a -section a guard cut does, wherever near its named offset the section is found — unless what the values read, and data -whose length the file gives, say makes the section unreadable whatever the cut hides: a fault of the file, as read -whole. +section a guard cut does, wherever near its named offset the section is found — unless what the values read say makes +the section unreadable whatever the cut hides, or data whose length the file gives, its filters written direct, lacks +the rows they declare whatever it hides: a fault of the file, as read whole, such a section read as read whole, lacking +rows. What a guard leaves out of the cross-reference index — the oldest sections of a long chain, the rows past a long table's bound, the objects past the most the index holds — is no damage either. Those objects read as null, unknown diff --git a/docs/website/docs/concepts/validation.md b/docs/website/docs/concepts/validation.md index fb0949e8..f99928f5 100644 --- a/docs/website/docs/concepts/validation.md +++ b/docs/website/docs/concepts/validation.md @@ -157,8 +157,9 @@ information, that it could not check it whole, rather than calling the file brok limit lets it check the rest. The same holds of a value that leads to such an object: a cross-reference stream whose `/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, leads to one is said not checked rather than malformed — so is one whose `/Length`, which the chain could not read, leads to one where only that length could say -where its data ends —, unless what the values read, and data whose length the file gives, say makes it unreadable whatever the guard hid, -which is the file's; +where its data ends —, unless what the values read say makes it unreadable whatever the guard hid, or data whose length the file gives, its +filters written direct, lacks the rows they declare whatever it hid — the file's, such a section read as read whole, +lacking rows; a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain there rather than naming a section not found; and an object stream whose `/Filter` or `/DecodeParms` leads to one, or to an object a limit left out of the index, is left encoded, the objects it holds said not checked rather than lost diff --git a/docs/website/docs/reference/diagnostics.md b/docs/website/docs/reference/diagnostics.md index 4db2c4d9..64e075da 100644 --- a/docs/website/docs/reference/diagnostics.md +++ b/docs/website/docs/reference/diagnostics.md @@ -102,7 +102,7 @@ Codes are stable: they are part of the public contract, because callers filter o | `xref.rebuilt` | The index was rebuilt by scanning the whole file | | `xref.offset-adjusted` | An object, or a cross-reference section the chain names, was not where the file said, and was found nearby | | `xref.section-missing` | A cross-reference section `/Prev` or `/XRefStm` names holds nothing that reads as a section, nor does any place nearby, lies outside the file, or is named by a value that is not an offset; what only it indexed is found by rebuilding the index when it is asked for. Reported at the offset the section was named at, or, when that offset lies outside the file or the value is not an offset, at the section whose trailer named it. A `/Prev` or `/XRefStm` the reader follows — not a hybrid file's cross-reference stream's own `/Prev`, nor the links of a table an `/XRefStm` names, which it leaves unread —, written as a reference that leads to an object one of the reader's limits cut, names what the guard hides: no section is reported missing, nothing is rebuilt, the chain ends there, and the guard is reported once the chain is read | -| `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, data or — with no `/Index` — `/Size` cannot give rows, or whose `/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, is a reference no section read before it places where it can be read without loading it, or whose `/Length` is such a reference, or leads to one through others, and no `endstream` follows its data before the next object the sections read so far place, or before the end of the file, or none was looked for once the document's searches for `endstream` read as much of the file as they may — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message. A value that names an object one of the reader's limits cut, directly or through a chain of references — a `/Length` whose data no `endstream` ends before that stop among them —, is the guard's, not the file's: the section is not reported, nor looked for nearby, the chain ends there, nothing is rebuilt, what only that section and the older ones index reads as null, and the guard is reported once the chain is read — but for what makes the section unreadable whatever the value holds, which is the file's and reported here, as read whole: a value past it no section read so far places, a `/W` that gives no three widths or none past 0, or rows numbered wrongly as far as the values read say — a subsection wrong by the element read whatever the cut one holds among them —, or data whose length the file gives, decoded whole, that holds fewer rows than the counts read declare however few bytes the widths read leave a row, a section then not looked for nearby | +| `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, data or — with no `/Index` — `/Size` cannot give rows, or whose `/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, is a reference no section read before it places where it can be read without loading it, or whose `/Length` is such a reference, or leads to one through others, and no `endstream` follows its data before the next object the sections read so far place, or before the end of the file, or none was looked for once the document's searches for `endstream` read as much of the file as they may — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message. A value that names an object one of the reader's limits cut, directly or through a chain of references — a `/Length` whose data no `endstream` ends before that stop among them —, is the guard's, not the file's: the section is not reported, nor looked for nearby, the chain ends there, nothing is rebuilt, what only that section and the older ones index reads as null, and the guard is reported once the chain is read — but for what makes the section unreadable whatever the value holds, which is the file's and reported here, as read whole: a value past it no section read so far places, a `/W` that gives no three widths or none past 0, or rows numbered wrongly as far as the values read say — a subsection wrong by the element read whatever the cut one holds among them —; data whose length the file gives, its filters written direct, decoded whole, that holds fewer rows than the counts read declare however few bytes the widths read leave a row is read as read whole instead, lacking rows, its trailer merged and the chain going on, where it was moved to too | | `xref.chain-cycle` | The chain of previous sections looped: a `/Prev` or `/XRefStm` names an offset the chain has already named, whether it read a section there or not. Reported at that offset, or, when it lies outside the file, at the section whose trailer named it | | `xref.entry-out-of-range` | An object's entry places it outside the file — past its end, before its start, or past what a `long` holds once the header's offset is added —; reported with no position, the offset in the message, exactly. A section named outside the file is `xref.section-missing`'s, or the rebuild's when `startxref` names it | | `stream.length-invalid` | A stream's `/Length` is not where its data ends: its `endstream` lies elsewhere and ends the data; or the `/Length` is no length — absent, not a non-negative integer, or naming an object the file lacks or that could not be read — and the `endstream` ends the data; or no `endstream` lies before the next object or the end of the file, no `endobj` of its object ends the data, and the data is the declared length, when it ends before that stop, or runs up to the next object; or the document's searches for `endstream` have read as much of the file as they may, and the data is taken without a search — but where a guard hid the `/Length`, whose data is then the guard's. A `/Length` that names an object a guard cut, or left out of the index, is not judged, though a missing `endstream` is reported all the same. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case. A cross-reference stream whose `/Length` the chain could not read, its data taken up to its first `endstream`, has it read once the chain is, unless a guard cut that data before every row the section declares — data that holds them all is checked whatever a guard did —: a `/Length` that is no length, or that no `endstream` confirms, is reported as any stream's, and one an `endstream` past the one the chain stopped at confirms is reported with the rows the chain left unread. Where the section's own row for that `/Length` lies among the rows the data holds, and gives a length an `endstream` confirms, the data is taken again to it before the rows are read, and nothing is reported; nor is anything of a `/Length` a guard cut or left out of the index | @@ -190,9 +190,9 @@ it —, or an `endstream` the data spells, an `endobj` after it or not, leaves t reported of the stream, whose data is cut by the guard, and whether its object ends with `endobj` is not judged. An `endstream` is taken for the stream's own, its end the file's, only where what follows its object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places, a cross-reference stream the -chain read, the end of the file, or the `xref` or `startxref` keyword. White space before the keyword, an end-of-line -above all, which only the hidden length says is the data's or not, leaves where the data ends to the guard as it -decodes: a lost tail, a missing checksum or end marker there is not reported, though a whole read reports it. +chain read, the end of the file, or the `xref` or `startxref` keyword. White space or comments before the keyword, an +end-of-line above all, which only the hidden length says are the data's or not, leave where the data ends to the guard +as it decodes, as [Damaged stream data](#damaged-stream-data) says. ```text Warning stream.length-invalid at 5501: The stream declared 19954 bytes but ended after 19952. @@ -258,13 +258,14 @@ data within `MaxStreamLength`, or its object within `MaxObjectLength`, or a filt hexadecimal digit without its pair or an ASCII85 group short of five is dropped rather than completed, and a predictor undoes the part of its last row the data holds rather than report rows the parameters describe as longer than the data. Nor is the end of data whose end a guard hides — its `endstream` proven its own past white space or comments that only -a `/Length` a guard cut or left out of the index says are the data's or not — reported. Where its first filter ends it -as its format says, with nothing to report, the data decodes as read whole; otherwise the filter decodes it up to where -that run starts: what it meets before is the file's and reported — a checksum that disagrees with the bytes before it -among it —, its running out there — a lost tail, a checksum short, a missing end marker — is not, and where its output -may lack what the run holds — Flate that lost its tail, LZW or RunLength data without its end marker — the output's end -is cut, as above, for the predictor, the filters after it and an object stream's members. The filters after a first that -ended whole decode as read whole, and report as read whole. +a `/Length` a guard cut or left out of the index says are the data's or not — reported. Its first filter decodes it up +to where that run starts — a comment past white space within the data, but in ASCII85 data, where a `%` is a digit —: +what it meets before is the file's and reported, its running out there — a lost tail, a checksum short, a missing end +marker — is not, and where its output may lack what the run holds — Flate that lost its tail, LZW or RunLength data +without its end marker — the output's end is cut, as above, for the predictor, the filters after it and an object +stream's members. Flate data whose deflate body ends in the data as taken decodes whole, what it reports as read whole +but a checksum that disagrees only where it reaches into the run. The filters after a first whose output is whole +decode, and report, as read whole. ```text Warning filter.failed at 59534: A Flate stream ends before its data does; what decoded before the end was kept. diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index 08b29cd1..1150ac2a 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -147,9 +147,10 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. `/W` that gives no three widths or none past 0, rows numbered wrongly — an `/Index` that holds no pairs, a subsection whose elements are no integers, whose count is below 0, or that numbers past the object numbers, by both its elements or by the one read whatever the cut one holds, a `/Size` with no `/Index` that gives no count —, as far as - the values read, written direct or through references, say, or data whose length the file gives, decoded whole, that - holds fewer rows than the counts read declare however few bytes the widths read leave a row: such a section is not - looked for nearby. The rows a cross-reference stream's data lacks, + the values read, written direct or through references, say. Data whose length the file gives, its filters written + direct, decoded whole, that holds fewer rows than the counts read declare however few bytes the widths read leave a + row lacks them whatever the cut hides: the section is read as read whole, lacking rows, its trailer merged, the chain + going on and the index rebuilt. The rows a cross-reference stream's data lacks, where its `/Length` so leads and its data was taken to the first `endstream`, are the guard's too, unless what follows that `endstream` proves it the stream's own, as above: they are then the file's to lack, and the index is rebuilt. An object the chain read through a reference the cut left whole is loaded once the chain is read only for a value it @@ -204,7 +205,7 @@ the documents producers write. They are not options: |---|---|---| | Nesting of arrays and dictionaries | 128 levels | Real documents nest a handful of levels; the bound keeps a hostile file from exhausting the stack | | A reference that resolves to another reference | followed 32 times | Producers write the object itself; the bound breaks a chain that loops | -| Searches for a stream's `endstream` | four times the file's length, for a document's searches together | A valid stream's `endstream` follows its declared length: the file is read past the bytes an object is parsed through only for the data of a cross-reference stream whose `/Length` the chain cannot read, for the data of a stream whose `/Length` a guard cut or left out of the index, and for white space and comments after a declared length that run past a first read — about twice the file at most —, and the proof that an `endstream` found under a hidden `/Length` is the stream's own reads 48 or 64 bytes past each, once, and the white space it crosses once more at most, about three times the file in all. Past the bound, a stream whose `endstream` the bytes read with its object do not show is not searched for further, and its report says so — or, where a guard hid its `/Length`, its data up to the next object is the guard's, `CutByGuard`, nothing reported | +| Searches for a stream's `endstream` | four times the file's length, for a document's searches together | A valid stream's `endstream` follows its declared length: the file is read past the bytes an object is parsed through only for the data of a cross-reference stream whose `/Length` the chain cannot read, for the data of a stream whose `/Length` a guard cut or left out of the index, and for white space and comments after a declared length that run past a first read — about twice the file, three times over the white space a cross-reference stream's search takes as data that its length, once read, is read on through —, and the proof that an `endstream` found under a hidden `/Length` is the stream's own reads 48 or 64 bytes past each, once, and the white space it crosses once more at most, about three times the file in all. Past the bound, a stream whose `endstream` the bytes read with its object do not show is not searched for further, and its report says so — or, where a guard hid its `/Length`, its data up to the next object is the guard's, `CutByGuard`, nothing reported | | Objects loaded while loading another | 64 | An indirect `/Length` leads to an integer, and a stream resolves its filters and the parameters they read before decoding any of its data: no level holds decoded data while the next loads | | Object numbers | up to 2,147,483,647 | An object's number is an `int`, and ISO 32000-1's Annex C advises at most 8,388,607 indirect objects. A number past it is refused where it is read — a header, a reference, a subsection, a row, an object stream's header —, never narrowed to one that would name another object. An object is numbered from 1; 0 names the head of the free list, where a reference, a subsection or a row may give it | | An object stream's members | each read to the next one's start, the last to the end of the decoded data | ISO 32000-1 (7.5.7) has a header list its objects by increasing offset, so that each lies before the next one starts: a pair whose offset is not past the one before it, or whose object would start past the data, ends the header there, as a stray token does, and the objects listed before it are read. A member past `MaxObjectLength` is that limit's | diff --git a/docs/website/docs/reference/validation-rules.md b/docs/website/docs/reference/validation-rules.md index b980e1f5..281c9baf 100644 --- a/docs/website/docs/reference/validation-rules.md +++ b/docs/website/docs/reference/validation-rules.md @@ -51,7 +51,7 @@ References are to ISO 32000-1:2008. | Rule | Severity | Meaning | Reads | |---|---|---|---| -| `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]); or a cross-reference stream whose `/Length` is such a reference, or leads through references to an object no section read before it places, and whose data no `endstream` ends before the next object the sections read so far place, or the end of the file — or was not searched for one, the document's searches for `endstream` having read as much of the file as they may ([#220]). Rows that spell `endstream` end such a stream's data there, and the section lacks the rows past them, unless the section's own row for that `/Length` — the row the reading of its rows keeps, one of a reserved type ignored —, among the rows before them, places an object that gives a longer length an `endstream` confirms: the data is then taken to it. A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. Nor is a cross-reference stream whose value the rows are read with leads to an object a limit cut: the section counts as cut there, the chain ends at it, nothing is rebuilt, and `xref.checked-in-part` says so ([#263]) — but for what makes it unreadable whatever the value holds, as read whole: a value past it no section read so far places, a `/W` that gives no three widths or none past 0, or rows numbered wrongly as far as the values read say — a subsection wrong by the element read whatever the cut one holds among them —, or data whose length the file gives, decoded whole, that holds fewer rows than the counts read declare however few bytes the widths read leave a row, whatever type the stream gives. Nor is one whose `/Length`, which the chain could not read, leads to an object `MaxObjectLength` cuts, where only that length could say where the data ends: with no `endstream` before the next object or the end of the file, the section counts as cut, as above; when its rows spell one, the integer the cut left of that object — whether the section's own row places it, or a newer section does — is where an `endstream` is looked for, the data taken to it when one confirms it, and otherwise the rows past the spelled one the limit's, the section counting as cut there, as above ([#224]). | Each section of the chain | +| `xref.section-malformed` | Error | A cross-reference section the chain names is there — an `xref` keyword, or a cross-reference stream — and cannot be read, or holds fewer rows than it declares: a subsection header that is not two integers or that numbers objects past 2,147,483,647, a row that is not an offset, a generation and `n` or `f`, a `/W` that is not three field widths that are not negative and not all zero, an `/Index` that is not pairs numbering objects from 0 to 2,147,483,647, a `/Size` that is no count of objects where there is no `/Index`, or a value the rows are read with — `/W`, `/Index`, `/Filter`, `/DecodeParms` and the parameters a Flate or LZW step reads, a `/Size` where there is no `/Index` — written as a reference no section read before it places where it can be read without loading it, or as references that lead from one to the next to no value ([#182]); or a cross-reference stream whose `/Length` is such a reference, or leads through references to an object no section read before it places, and whose data no `endstream` ends before the next object the sections read so far place, or the end of the file — or was not searched for one, the document's searches for `endstream` having read as much of the file as they may ([#220]). Rows that spell `endstream` end such a stream's data there, and the section lacks the rows past them, unless the section's own row for that `/Length` — the row the reading of its rows keeps, one of a reserved type ignored —, among the rows before them, places an object that gives a longer length an `endstream` confirms — in a predicted row the spelled one parts too, undone as far as it goes —: the data is then taken to it. A row that gives an object in use what no entry can hold — a generation past 65,535, an offset of 2⁶³ or more, an object stream that is no object number, an index past 2,147,483,647, or a field wider than 8 bytes whose leading bytes are not all zero — is refused alone, and makes the section malformed too; a free row is read whatever its generation, and a row of a type past 2 is skipped ([#221]). The objects it leaves out are found, if at all, by scanning the file. A section one of the reader's limits cut is not malformed for the cut (ADR 34), but what the reader read of it is the file's: a row it refused before the cut, what it met past the rows reading on to a table's trailer. Nor is a cross-reference stream whose value the rows are read with leads to an object a limit cut: the section counts as cut there, the chain ends at it, nothing is rebuilt, and `xref.checked-in-part` says so ([#263]) — but for what makes it unreadable whatever the value holds, as read whole: a value past it no section read so far places, a `/W` that gives no three widths or none past 0, or rows numbered wrongly as far as the values read say — a subsection wrong by the element read whatever the cut one holds among them —, or data whose length the file gives, its filters written direct, decoded whole, that holds fewer rows than the counts read declare however few bytes the widths read leave a row — read then as read whole, lacking rows —, whatever type the stream gives. Nor is one whose `/Length`, which the chain could not read, leads to an object `MaxObjectLength` cuts, where only that length could say where the data ends: with no `endstream` before the next object or the end of the file, the section counts as cut, as above; when its rows spell one, the integer the cut left of that object — whether the section's own row places it, or a newer section does — is where an `endstream` is looked for, the data taken to it when one confirms it, and otherwise the rows past the spelled one the limit's, the section counting as cut there, as above ([#224]). | Each section of the chain | | `xref.section-not-found` | Error | A section a `/Prev` or an `/XRefStm` names is neither there nor within 512 bytes of it, or what names it is not an offset — tiff2pdf writes `/Prev 576066 0 R`. The objects only it indexes are missing from the index. The section `startxref` names is `file.startxref-wrong`'s. A `/Prev` or an `/XRefStm` the reader follows, written as a reference that leads to an object `MaxObjectLength` cuts, names no section missing: the chain ends there for the limit, what it names unread, which `xref.checked-in-part` says ([#263]). | Each section of the chain | | `xref.section-shifted` | Warning | A section a `/Prev` or an `/XRefStm` names starts a few bytes from where it is named, and the reader found it there. | Each section of the chain | | `xref.chain-loop` | Error | The chain names a section it has already read, or an offset it has already named where it read none, in the file or outside it: the reader stops there, and the section the chain should have gone on to is unknown, what only it indexed found, if at all, by rebuilding the index — as for `xref.section-not-found`. | The chain | diff --git a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs index 961d06cf..76456305 100644 --- a/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs +++ b/src/AdCodicem.Pdf/IO/Filters/PdfFilterPipeline.cs @@ -40,10 +40,10 @@ public static ReadOnlyMemory Decode(PdfStream stream, PdfDiagnostics? diag /// the reader lets data pass through, or a value the filters read that a guard cut or left out of the index: what comes /// back is then the data still encoded by the filters past it, no part of what the stream decodes to —, and whether one /// cut the data, as a reader of its rows or its objects must know. Data a proven endstream ended past white space - /// and comments that only a hidden length says are the data's or not () decodes as - /// read whole where its first filter ends it as its format says; otherwise up to where that run starts, a filter that - /// runs out there saying nothing of the file, and the data cut by a guard where the first filter's output may lack what - /// the run holds. + /// and comments that only a hidden length says are the data's or not () decodes up + /// to where that run starts, the first filter's running out there saying nothing of the file, and the data cut by a + /// guard where its output may lack what the run holds; Flate data whose deflate body ends in the data as taken is + /// whole. /// /// The stream to decode. /// Receives what decoding met; when none are supplied, the stream's document does. diff --git a/src/AdCodicem.Pdf/Objects/PdfStreamData.cs b/src/AdCodicem.Pdf/Objects/PdfStreamData.cs index 85e36463..cf3c94c5 100644 --- a/src/AdCodicem.Pdf/Objects/PdfStreamData.cs +++ b/src/AdCodicem.Pdf/Objects/PdfStreamData.cs @@ -34,9 +34,9 @@ public abstract class PdfStreamData /// /// Gets a value indicating whether where the data ends is the guard's to hide, though the data is the file's: an /// endstream proven the stream's own ended it under a /Length a guard hid, with white space, or comments, - /// before the keyword that only the length could say are the data's or not (#224). The data decodes as read whole where - /// its first filter ends it as its format says; otherwise up to where that run starts, what lies before it the file's, - /// a filter that runs out there saying nothing of the file, and the end of what it may lack the guard's. + /// before the keyword that only the length could say are the data's or not (#224). Its first filter decodes it up to + /// where that run starts, what lies before it the file's, its running out there saying nothing of the file, and the + /// end of what it may lack the guard's; Flate data whose deflate body ends in the data as taken is whole. /// internal virtual bool EndHidden => false; From 42be42923376be8d347fab20f9029fde95418fd3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 07:08:52 +0000 Subject: [PATCH 137/146] fix: end the chain at a section lacking rows whose data may hold rows a guard hid A cross-reference stream whose data cannot hold the rows its values declare, beside a value a guard cut, was read as read whole: its trailer merged and the chain going on. The rows its data held, read with the cut value, were not read, and an older section's rows stood for them, so a superseded catalog was served. The cut value was never loaded, so the guard was neither reported nor thrown under ThrowOnLimit, and the validator charged the file with a wrong /Size the hidden rows would have set right. The cut value is now loaded once the chain is read. Where the data may hold a row at the fewest bytes a row may take, the section is also the guard's: the chain ends there, as where rows a guard cut, the size rule leaves it unjudged and the validator says it went unchecked. Data that holds no row leaves the chain going on. Refs #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 30 +++- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 167 ++++++++++++++++-- 2 files changed, 173 insertions(+), 24 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 32874673..da69238b 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -3988,21 +3988,33 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect return XRefSectionState.Malformed; } - if (FaultBesideCut(dictionary, deferred is null && !xrefStream.Data.CutByGuard && !xrefStream.Data.EndHidden ? xrefStream : null, out var lacking) is { } file) + if (FaultBesideCut(dictionary, deferred is null && !xrefStream.Data.CutByGuard && !xrefStream.Data.EndHidden ? xrefStream : null, out var holds) is { } file) { section.Fault = file; - if (!lacking) + if (holds is not { } rows) { return XRefSectionState.Malformed; } // Data whose extent the file gives that cannot hold the rows the values read declare lacks rows whatever the - // cut value holds: the section is read as read whole, the file's to lack them — its trailer merged, the chain - // going on, the index rebuilt —, though the rows it holds, which the cut value is read with, are not read. + // cut value holds: the section is read as read whole, the file's to lack them — its trailer merged, the index + // rebuilt —, and the guard is reached once the chain is read. The rows the data may hold, at the fewest bytes + // a row may take, are read with the cut value: the guard's, which a whole read reads, the chain ending here, as + // where rows a guard cut, since an older section's rows would stand for them. Data that holds none leaves the + // chain going on. _indexIncomplete = true; section.Incomplete = true; _xref.MergeTrailer(dictionary); + (_chainValuesCut ??= []).Add(cut); + + if (rows > 0) + { + _cutByGuard = true; + section.CutByLimit = true; + return XRefSectionState.Read; + } + string? hiddenPrevious = null; previous = section.NamedBy == "/XRefStm" ? -1 : SectionOffset(dictionary, PdfName.Prev, absolute, out hiddenPrevious); section.ChainHiddenByGuard = hiddenPrevious; @@ -4516,10 +4528,12 @@ private XRefSectionState CutByValue(XRefSectionRecord section, PdfObjectId cut) /// /// The stream's dictionary. /// The stream, whose data's extent is the file's, or null when it is not. - /// Whether the fault is that the data holds fewer rows than the section declares. - private string? FaultBesideCut(PdfDictionary dictionary, PdfStream? stream, out bool lacking) + /// + /// The most rows the data holds, where the fault is that it holds fewer than the section declares; null otherwise. + /// + private string? FaultBesideCut(PdfDictionary dictionary, PdfStream? stream, out long? held) { - lacking = false; + held = null; // The fewest bytes a row may hold, which the widths read give, and the fewest rows the counts read declare. long row = 0; @@ -4622,7 +4636,7 @@ private XRefSectionState CutByValue(XRefSectionRecord section, PdfObjectId cut) return null; } - lacking = true; + held = most.Rows; return string.Create( CultureInfo.InvariantCulture, $"its data, {most.Length:N0} {(most.Length == 1 ? "byte" : "bytes")} decoded, holds at most {most.Rows:N0} of the {declared:N0}{(exact ? string.Empty : " or more")} {(declared == 1 && exact ? "row" : "rows")} its /Index and /Size declare"); diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 51aeb2f3..10450a70 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -3284,20 +3284,20 @@ public void A_stream_whose_value_a_guard_cut_where_the_chain_names_a_section_is_ } [Theory] - [InlineData("<< /Type /Font /W 5 0 R /Size 8 /Length 3 >>\nstream\nabc\nendstream", "[1 4 2]", "/Prev", "holds at most 3 of the 8 rows")] - [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Length 3 /Info 8 0 R /ID [<01> <02>] >>\nstream\nabc\nendstream", "[1 4 2]", "/Prev", "holds at most 3 of the 8 rows")] - [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Length 3 >>\nstream\nabc\nendstream", "[1 4 2]", "/Prev+2", "holds at most 3 of the 8 rows")] - [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Length 3 >>\nstream\nabc\nendstream", "[1 4 2]", "/XRefStm", "holds at most 3 of the 8 rows")] - [InlineData("<< /Type /Xref /W [1 5 0 R 2] /Size 8 /Length 3 >>\nstream\nabc\nendstream", "4", "/Prev", "holds at most 1 of the 8 rows")] - [InlineData("<< /Type /XRef /W [1 4 2] /Index [5 0 R 8] /Length 3 >>\nstream\nabc\nendstream", "0", "/Prev", "its data, 3 bytes decoded, holds at most 0 of the 8 rows")] - [InlineData("<< /Type /XRef /W [1 4 2] /Index [5 0 R 1] /Length 1 >>\nstream\na\nendstream", "0", "/Prev", "its data, 1 byte decoded, holds at most 0 of the 1 row its")] - [InlineData("<< /Type /XRef /W [1 4 2] /Index [5 0 R 1 9 5 0 R] /Length 3 >>\nstream\nabc\nendstream", "0", "/Prev", "holds at most 0 of the 1 or more rows")] - [InlineData("<< /Type /XRef /W [1 4 2] /Index [5 0 R 2147483648] /Length 56 >>\nstream\n{56}\nendstream", "0", "/Prev", "holds at most 8 of the 2,147,483,648 rows")] - [InlineData("<< /Type /XRef /W [1 9223372036854775807 5 0 R] /Size 8 /Length 56 >>\nstream\n{56}\nendstream", "0", "/Prev", "holds at most 0 of the 8 rows")] - [InlineData("<< /Type /XRef /W [4611686018427387904 4611686018427387904 5 0 R] /Size 8 /Length 56 >>\nstream\n{56}\nendstream", "0", "/Prev", "holds at most 0 of the 8 rows")] - [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Filter /FlateDecode /Length {length} >>\nstream\n{flate}\nendstream", "[1 4 2]", "/Prev", "its data, 5 bytes decoded, holds at most 5 of the 8 rows")] + [InlineData("<< /Type /Font /W 5 0 R /Size 8 /Length 3 >>\nstream\nabc\nendstream", "[1 4 2]", "/Prev", "holds at most 3 of the 8 rows", true)] + [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Length 3 /Info 8 0 R /ID [<01> <02>] >>\nstream\nabc\nendstream", "[1 4 2]", "/Prev", "holds at most 3 of the 8 rows", true)] + [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Length 3 >>\nstream\nabc\nendstream", "[1 4 2]", "/Prev+2", "holds at most 3 of the 8 rows", true)] + [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Length 3 >>\nstream\nabc\nendstream", "[1 4 2]", "/XRefStm", "holds at most 3 of the 8 rows", true)] + [InlineData("<< /Type /Xref /W [1 5 0 R 2] /Size 8 /Length 3 >>\nstream\nabc\nendstream", "4", "/Prev", "holds at most 1 of the 8 rows", true)] + [InlineData("<< /Type /XRef /W [1 4 2] /Index [5 0 R 8] /Length 3 >>\nstream\nabc\nendstream", "0", "/Prev", "its data, 3 bytes decoded, holds at most 0 of the 8 rows", false)] + [InlineData("<< /Type /XRef /W [1 4 2] /Index [5 0 R 1] /Length 1 >>\nstream\na\nendstream", "0", "/Prev", "its data, 1 byte decoded, holds at most 0 of the 1 row its", false)] + [InlineData("<< /Type /XRef /W [1 4 2] /Index [5 0 R 1 9 5 0 R] /Length 3 >>\nstream\nabc\nendstream", "0", "/Prev", "holds at most 0 of the 1 or more rows", false)] + [InlineData("<< /Type /XRef /W [1 4 2] /Index [5 0 R 2147483648] /Length 56 >>\nstream\n{56}\nendstream", "0", "/Prev", "holds at most 8 of the 2,147,483,648 rows", true)] + [InlineData("<< /Type /XRef /W [1 9223372036854775807 5 0 R] /Size 8 /Length 56 >>\nstream\n{56}\nendstream", "0", "/Prev", "holds at most 0 of the 8 rows", false)] + [InlineData("<< /Type /XRef /W [4611686018427387904 4611686018427387904 5 0 R] /Size 8 /Length 56 >>\nstream\n{56}\nendstream", "0", "/Prev", "holds at most 0 of the 8 rows", false)] + [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Filter /FlateDecode /Length {length} >>\nstream\n{flate}\nendstream", "[1 4 2]", "/Prev", "its data, 5 bytes decoded, holds at most 5 of the 8 rows", true)] public void A_section_whose_data_holds_fewer_rows_than_it_declares_whatever_a_value_a_guard_cut_holds_is_the_file_s_to_lack( - string six, string five, string naming, string fault) + string six, string five, string naming, string fault, bool hidesRows) { // As above, object 6's data, whose length it gives, holds fewer rows than its /Size or the /Index counts read declare, // however few bytes the widths read and the one cut give a row — a width past what data holds counted as such —: @@ -3353,16 +3353,151 @@ public void A_section_whose_data_holds_fewer_rows_than_it_declares_whatever_a_va var section = document.Reader.Structure.Sections[^1]; section.Offset.Should().Be(sixAt); section.State.Should().Be(naming == "/Prev+2" ? XRefSectionState.Relocated : XRefSectionState.Read); - section.CutByLimit.Should().BeFalse(); + section.CutByLimit.Should().Be(hidesRows); section.Incomplete.Should().BeTrue(); section.Fault.Should().Contain(fault); document.WasRepaired.Should().BeTrue(); document.GetObject(new PdfObjectId(1)).AsDictionary().Required().IsOfType(PdfName.Catalog).Should().BeTrue(); document.Trailer.Keys.Select(key => key.Value).Order().Should().Equal(wholeKeys); - // The rebuild reaches the object the guard cut, as it reaches every object. + // The guard is reached once the chain is read. Rows the data may hold, which the guard hid, are said unchecked, and + // the highest number the chain uses, which they may raise, is not judged. + document.Diagnostics.Select(entry => entry.Code).Should().Contain(PdfDiagnosticCodes.LimitObject); document.Diagnostics.Select(entry => entry.Code).Where(code => code != PdfDiagnosticCodes.LimitObject).Should().Equal(wholeCodes); - new PdfValidator().Validate(document).Findings.Select(finding => finding.RuleId).Distinct().Order().Should().Equal(wholeRules); + new PdfValidator().Validate(document).Findings.Select(finding => finding.RuleId).Distinct().Order().Should().Equal( + hidesRows + ? [.. wholeRules.Where(rule => rule != PdfValidationRuleIds.FileSizeWrong).Append(PdfValidationRuleIds.XRefCheckedInPart).Order()] + : wholeRules); + } + + [Theory] + [InlineData("/Prev")] + [InlineData("/XRefStm")] + public void Rows_a_section_lacking_rows_holds_beside_a_value_a_guard_cut_are_the_guard_s_and_no_older_section_stands_for_them( + string naming) + { + // Stream 6, of the second revision, places the catalog's newer definition in the seven rows its data holds of the + // eight its /Index declares, whose first number names object 5, which MaxObjectLength cuts; the first revision's + // table places the older one. Read whole, its rows win. The rows the data holds are read with the value the guard + // cut: the guard's, as rows a guard cut are, the chain ending there, where an older section's rows would stand for + // them — the older catalog was served, and the guard neither reported nor thrown under ThrowOnLimit. + var file = LackingBesideCut(naming, "/Index [5 0 R 8] /Size 8", rows => string.Concat(rows[..7]), out var sixAt); + + using (var whole = PdfDocument.Open(file)) + { + whole.WasRepaired.Should().BeFalse(); + whole.Reader.Structure.Sections.Single(section => section.Offset == sixAt).Incomplete.Should().BeTrue(); + Revision(whole).Should().Be("two"); + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal( + PdfDiagnosticCodes.LimitObject, PdfDiagnosticCodes.XRefRebuilt, PdfDiagnosticCodes.ObjectRedefined); + document.Reader.Structure.Sections.Should().HaveCount(2); + var section = document.Reader.Structure.Sections[^1]; + section.Offset.Should().Be(sixAt); + section.State.Should().Be(XRefSectionState.Read); + section.CutByLimit.Should().BeTrue(); + section.Incomplete.Should().BeTrue(); + section.Fault.Should().Contain("holds at most 7 of the 8 rows"); + Revision(document).Should().Be("two"); + + var throwing = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); + throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); + } + + [Theory] + [InlineData("/Prev")] + [InlineData("/XRefStm")] + public void A_section_lacking_rows_beside_a_value_a_guard_cut_reaches_the_guard_once_the_chain_is_read(string naming) + { + // As above, but stream 6's data, three bytes, holds no row of seven bytes: the section lacks rows whatever the value + // holds, its trailer merged and the chain going on, the first revision's table indexing every object, so that + // nothing is rebuilt. The value the guard cut was never loaded: the guard was neither reported nor thrown. + var file = LackingBesideCut(naming, "/Index [5 0 R 8] /Size 8", _ => "abc", out var sixAt); + string[] wholeCodes; + + using (var whole = PdfDocument.Open(file)) + { + whole.Reader.Structure.Sections.Single(section => section.Offset == sixAt).Incomplete.Should().BeTrue(); + wholeCodes = [.. whole.Diagnostics.Select(entry => entry.Code)]; + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal([.. wholeCodes, PdfDiagnosticCodes.LimitObject]); + document.Reader.Structure.Sections.Should().HaveCount(3); + var section = document.Reader.Structure.Sections.Single(section => section.Offset == sixAt); + section.CutByLimit.Should().BeFalse(); + section.Incomplete.Should().BeTrue(); + Revision(document).Should().Be("one"); + + var throwing = () => PdfDocument.Open(file, ObjectGuard with { ThrowOnLimit = true }); + throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); + } + + [Fact] + public void The_validator_charges_the_file_with_no_size_rows_a_section_lacking_rows_holds_beside_a_value_a_guard_cut_hide() + { + // Stream 6 places object 9 in the three rows its data holds of the five its /Index declares, whose first number a guard + // cut: read whole, the highest number the chain uses is 9, and the /Size 10 the trailers give is right. The rows the + // guard hid were not counted: the file was charged with a /Size two too large, and nothing said they went unchecked. + var file = LackingBesideCut( + "/Prev", "/Index [5 0 R 2 9 3] /Size 10", rows => rows[0] + rows[1] + rows[9], out _, tableSize: 10); + string[] wholeRules; + + using (var whole = PdfDocument.Open(file)) + { + wholeRules = [.. new PdfValidator().Validate(whole).Findings.Select(finding => finding.RuleId).Distinct().Order()]; + } + + wholeRules.Should().NotContain(PdfValidationRuleIds.FileSizeWrong); + + using var document = PdfDocument.Open(file, ObjectGuard); + + new PdfValidator().Validate(document).Findings.Select(finding => finding.RuleId).Distinct().Order().Should().Equal( + [.. wholeRules.Append(PdfValidationRuleIds.XRefCheckedInPart).Order()]); + } + + /// Gives the /Rev of the catalog reads, which says which revision it is. + private static string? Revision(PdfDocument document) => + (document.GetObject(new PdfObjectId(1)).AsDictionary().Required().GetRaw(PdfName.Get("Rev")) as PdfString)?.ToText(); + + /// + /// Builds three revisions: the first's catalog, /Rev (one), pages and table; the second's catalog, + /// /Rev (two), object 5 — 0, past MaxObjectLength —, object 9 and cross-reference stream 6, whose dictionary holds + /// and whose data gives from the rows that would place objects 0 to + /// 9; the third's table, placing object 5, which names stream 6 by . + /// + private static byte[] LackingBesideCut( + string naming, string numbering, Func data, out long sixAt, int tableSize = 8) + { + static string Row(int type, long offset, int generation) => string.Concat( + (char)type, (char)((offset >> 24) & 0xFF), (char)((offset >> 16) & 0xFF), (char)((offset >> 8) & 0xFF), + (char)(offset & 0xFF), (char)(generation >> 8), (char)(generation & 0xFF)); + + var file = new HandWrittenFile("", "1.5"); + var older = file.Object(1, "<< /Type /Catalog /Pages 2 0 R /Rev (one) >>"); + var two = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + var three = file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Resources << >> >>"); + var first = file.Table(4, new Dictionary { [1] = older, [2] = two, [3] = three }, "/Size 4 /Root 1 0 R"); + var one = file.Object(1, "<< /Type /Catalog /Pages 2 0 R /Rev (two) >>"); + var five = file.Object(5, "0" + new string(' ', 1100)); + var nine = file.Object(9, "(nine)"); + sixAt = file.Position; + var rows = data( + [ + Row(0, 0, 65535), Row(1, one, 0), Row(1, two, 0), Row(1, three, 0), Row(0, 0, 0), Row(1, five, 0), Row(1, sixAt, 0), + Row(0, 0, 0), Row(0, 0, 0), Row(1, nine, 0), + ]); + var prev = naming == "/Prev" ? Invariant($" /Prev {first}") : string.Empty; + file.Raw(Invariant($"6 0 obj\n<< /Type /XRef /W [1 4 2] {numbering} /Root 1 0 R{prev} /Length {rows.Length} >>\nstream\n{rows}\nendstream\nendobj\n")); + file.Raw("%" + new string('-', 1200) + "\n"); + var table = file.Position; + var link = naming == "/Prev" ? Invariant($"/Prev {sixAt}") : Invariant($"/Prev {first} /XRefStm {sixAt}"); + file.Raw(Invariant($"xref\n0 1\n0000000000 65535 f \n5 1\n{five:D10} 00000 n \ntrailer\n<< /Size {tableSize} /Root 1 0 R {link} >>\nstartxref\n{table}\n%%EOF\n")); + return file.Bytes; } [Fact] From bb197f5136a3c2845a985ce50b60c33e0f5e6ac0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 07:16:41 +0000 Subject: [PATCH 138/146] fix: find a deferred length's row in any predicted row a spelled endstream parts, guard kept A cross-reference stream whose /Length only its own rows place is taken to the first endstream its rows may spell. Where that endstream parts the predicted row holding the row for the /Length, the data was decoded once more knowing its end hidden only when the row lay past the rows the first decoding held. A PNG predictor whose first row is parted leaves the data as decoded, and the TIFF predictor leaves a parted row differenced: the row was read garbled, and a valid file's index was rebuilt. That decoding also dropped a decoding guard it reached, which ThrowOnLimit threw: the rows past the bound were charged to the file in one mode and thrown in the other. The data is now decoded once more wherever a subsection numbers the object and no row standing for it was found, and the row looked for in the rows that decoding holds. A guard that decoding reaches stands, and where it found no row the rows past the bound are the guard's, as where the first decoding reaches it. A test holds data that holds no whole row, which no row is looked for in. Refs #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 45 +++--- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 134 +++++++++++++++++- .../DocumentReaderTests.cs | 78 +++++++++- 3 files changed, 231 insertions(+), 26 deletions(-) diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index da69238b..3367ff97 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -4150,21 +4150,34 @@ private XRefSectionState TryReadXRefStream(long absolute, XRefSectionRecord sect if (!retaken && !endHidden && deferred is { } unsettled && declared > held && decodedWhole) { long[] ranges = index is null ? [0, declared] : subsections; - var whole = RetakeDeferred(xrefStream, unsettled, data, fieldWidths, row, held, ranges, out lengthCut, out var past); + var whole = RetakeDeferred(xrefStream, unsettled, data, fieldWidths, row, held, ranges, out lengthCut, out var numbered); // The row for the /Length may lie in a predicted row the spelled endstream parts, which the data decoded as - // data whose end the file gives dropped: the data decoded knowing its end hidden undoes it as far as it goes, - // and holds the rows before the endstream whole. What that decoding reports is dropped with it. - if (whole is null && past && row > 0) + // data whose end the file gives dropped, left as decoded — the first predicted row parted — or left still + // differenced: the data decoded knowing its end hidden undoes it as far as it goes, and holds the rows before + // the endstream whole. What that decoding reports is dropped with it, but for a guard on decoding it reached, + // which the whole data, holding at least as much, reaches too: where it found no row for the /Length, where + // the rows end is the guard's, as where the decoding before reached it. + if (whole is null && numbered && row > 0) { var again = _diagnostics.GetMarkWithDecodingReports(); - var undone = Filters.PdfFilterPipeline.Decode(xrefStream, _diagnostics, endHidden: true, reachedBefore: false, out var leftAgain, out _); - _diagnostics.RollBack(again); + var undone = Filters.PdfFilterPipeline.Decode(xrefStream, _diagnostics, endHidden: true, reachedBefore: false, out var leftAgain, out var reached); - if (!leftAgain && undone.Length / row > held) + if (!leftAgain) { whole = RetakeDeferred(xrefStream, unsettled, undone.Span, fieldWidths, row, undone.Length / row, ranges, out lengthCut, out _); } + + if (whole is null && reached) + { + _diagnostics.RollBackBesideLimits(decoding); + reachedBefore = true; + heldBefore = undone.Length / row; + endHidden = true; + continue; + } + + _diagnostics.RollBack(again); } if (whole is not null) @@ -4339,8 +4352,8 @@ private bool EndsAtOwnEndStream(PdfStream stream, ReadOnlySpan data, ReadO /// gives, which the chain could not read and took the data up to the first endstream for, when the rows /// holds show that endstream was in the rows (#220): the section's own row for the object /// the /Length names, among those held, places it, and the length it gives runs past the data taken, to an - /// endstream that confirms it. Null otherwise: the length is checked once the chain is read — and the row lies - /// past those held where past says so. + /// endstream that confirms it. Null otherwise: the length is checked once the chain is read — and a subsection + /// numbers the object, its row past those held or not standing, where says so. /// /// /// The section's row for the object is the first that numbers it with a type the reading of its rows takes: one of a @@ -4364,10 +4377,10 @@ private bool EndsAtOwnEndStream(PdfStream stream, ReadOnlySpan data, ReadO long held, long[] subsections, out bool lengthCut, - out bool past) + out bool numbered) { lengthCut = false; - past = false; + numbered = false; var number = deferred.Length.Number; if (_xref.TryGet(number, out _) || _refusedNumbers?.Contains(number) == true) @@ -4384,17 +4397,17 @@ private bool EndsAtOwnEndStream(PdfStream stream, ReadOnlySpan data, ReadO ulong type = 0; ulong offset = 0; ulong generation = 0; - var numbered = false; + var taken = false; - for (var i = 0; i < subsections.Length && !numbered; i += 2) + for (var i = 0; i < subsections.Length && !taken; i += 2) { if (number >= subsections[i] && number - subsections[i] < subsections[i + 1]) { var at = ordinal + (number - subsections[i]); + numbered = true; if (at >= held) { - past = true; return null; } @@ -4404,13 +4417,13 @@ private bool EndsAtOwnEndStream(PdfStream stream, ReadOnlySpan data, ReadO type = widths[0] == 0 ? 1 : ReadField(fields, ref cursor, widths[0]); offset = ReadField(fields, ref cursor, widths[1]); generation = ReadField(fields, ref cursor, widths[2]); - numbered = type <= 2; + taken = type <= 2; } ordinal += subsections[i + 1]; } - if (!numbered || type != 1 || offset > long.MaxValue || generation != (ulong)deferred.Length.Generation) + if (!taken || type != 1 || offset > long.MaxValue || generation != (ulong)deferred.Length.Generation) { return null; } diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 2455ed6a..10852922 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -858,9 +858,10 @@ byte[] Data(long five, long stream) /// whose /Length names object 5, which only its own rows place: the free row 0's, the catalog's, the page tree's /// and the page's, then object 5's, two of reserved types that spell endstream after an end-of-line, and its own /// (/Index [0 4 5 1 19 2 7 1]). With 28 columns, the spelled endstream parts the second predicted row - /// right after object 5's row. The file is valid. + /// right after object 5's row; with 49 or 56, the first, which object 5's row lies in. With 2, + /// the TIFF predictor differences each row in place of the PNG one. The file is valid. /// - public static byte[] DeferredLengthRowPartedBySpelledEndStream(int columns) + public static byte[] DeferredLengthRowPartedBySpelledEndStream(int columns, int predictor = 12) { using var file = new Writer("1.5"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); @@ -883,6 +884,26 @@ byte[] Data(long five, long stream) var predicted = new List(); + if (predictor == 2) + { + // Each byte but a row's first is written as its difference from the one before it: the two rows of reserved + // types hold the bytes whose differences spell the endstream, their types still past 2, which no reading + // takes. + var spelled = "\nendstream\0\0\0\0"u8; + + for (var i = 35; i < 49; i++) + { + rows[i] = (byte)((i % columns == 0 ? 0 : rows[i - 1]) + spelled[i - 35]); + } + + for (var i = 0; i < rows.Count; i++) + { + predicted.Add(i % columns == 0 ? rows[i] : (byte)(rows[i] - rows[i - 1])); + } + + return Compress([.. predicted]); + } + for (var i = 0; i < rows.Count; i += columns) { predicted.Add(0); @@ -898,7 +919,114 @@ byte[] Data(long five, long stream) 7, string.Create( CultureInfo.InvariantCulture, - $"/Type /XRef /Size 21 /W [1 4 2] /Index [0 4 5 1 19 2 7 1] /Filter /FlateDecode /DecodeParms << /Predictor 12 /Columns {columns} >> /Root 1 0 R /Length 5 0 R"), + $"/Type /XRef /Size 21 /W [1 4 2] /Index [0 4 5 1 19 2 7 1] /Filter /FlateDecode /DecodeParms << /Predictor {predictor} /Columns {columns} >> /Root 1 0 R /Length 5 0 R"), + Data(file.OffsetOf(5), at)); + file.StartXRef(at); + return file.ToArray(); + } + + /// + /// A file of one revision: the catalog, the page tree, the page, object 5, which holds the length of cross-reference + /// stream 7's data, and stream 7, raw, whose /Length names object 5, which only its own rows place. Its data + /// starts with zero bytes, fewer than a row's seven, then spells endstream after an + /// end-of-line, before its rows: the free row 0's, the catalog's, the page tree's, the page's, object 5's and its own + /// (/Index [19 2 0 4 5 1 7 1]). The file is valid. + /// + public static byte[] DeferredLengthEndStreamSpelledBeforeRows(int prefix) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + + // The data's length does not depend on the offsets its rows hold. + byte[] Data(long five, long stream) + { + var rows = new List(new byte[prefix]); + rows.AddRange("\nendstream\0"u8.ToArray()); + + while (rows.Count % 7 != 0) + { + rows.Add(0); + } + + rows.AddRange(Encode( + [(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0), (1, five, 0), (1, stream, 0)], + Rows.Raw)); + return [.. rows]; + } + + file.Object(5, Data(0, 0).Length.ToString(CultureInfo.InvariantCulture)); + var at = file.Position; + file.Stream(7, "/Type /XRef /Size 21 /W [1 4 2] /Index [19 2 0 4 5 1 7 1] /Root 1 0 R /Length 5 0 R", Data(file.OffsetOf(5), at)); + file.StartXRef(at); + return file.ToArray(); + } + + /// + /// A file of one revision: the catalog, the page tree, the page, object 5 and cross-reference stream 7, whose + /// /Length names object 5 — or, with , gives the length —, and whose data a RunLength + /// step decodes after a Flate one, in a stored block with a PNG predictor of 45 columns. The RunLength data: a literal run + /// of the rows of the free row 0, the catalog, the page tree and the page, 14 runs of 128 zeros — 256 free rows —, then + /// object 5's row and two of reserved types that spell endstream after an end-of-line — with + /// ; past them otherwise —, the stream's own row, and the end byte. The spelled + /// endstream parts the second predicted row, past the 1,052 bytes the data up to it decodes to, and object 5's row + /// lies 1,820 bytes or more into the rows. The file is valid. + /// + public static byte[] DeferredLengthRowPastRunLengthRuns(bool rowBefore, bool direct = false) + { + const int Columns = 45; + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + + // The data's length does not depend on the offsets its rows hold. + byte[] Data(long five, long stream) + { + var runs = new List { 27 }; + runs.AddRange(Encode([(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0)], Rows.Raw)); + + for (var i = 0; i < 14; i++) + { + runs.AddRange([0x81, 0x00]); + } + + if (rowBefore) + { + runs.Add(6); + runs.AddRange(Encode([(1, five, 0)], Rows.Raw)); + } + + runs.Add(13); + runs.AddRange("\nendstream\0\0\0\0"u8.ToArray()); + runs.Add(rowBefore ? (byte)6 : (byte)13); + runs.AddRange(Encode(rowBefore ? [(1, stream, 0)] : [(1, five, 0), (1, stream, 0)], Rows.Raw)); + + while (runs.Count % Columns != 0) + { + runs.Add(128); + } + + var predicted = new List(); + + for (var i = 0; i < runs.Count; i += Columns) + { + predicted.Add(0); + predicted.AddRange(runs.GetRange(i, Columns)); + } + + return Compress([.. predicted]); + } + + var length = Data(0, 0).Length.ToString(CultureInfo.InvariantCulture); + file.Object(5, direct ? "0" : length); + var at = file.Position; + file.Stream( + 7, + string.Create( + CultureInfo.InvariantCulture, + $"/Type /XRef /Size 266 /W [1 4 2] /Index {(rowBefore ? "[0 4 8 256 5 1 264 2 7 1]" : "[0 4 8 256 264 2 5 1 7 1]")} /Filter [/FlateDecode /RunLengthDecode] /DecodeParms [<< /Predictor 12 /Columns {Columns} >> null] /Root 1 0 R /Length {(direct ? length : "5 0 R")}"), Data(file.OffsetOf(5), at)); file.StartXRef(at); return file.ToArray(); diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 6aeb3dae..4294d3b1 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -3127,15 +3127,21 @@ protected override void Dispose(bool disposing) } [Theory] - [InlineData(28)] - [InlineData(21)] - public void Reads_a_cross_reference_stream_whose_length_row_a_spelled_endstream_parts_the_predicted_row_of(int columns) + [InlineData(28, 12)] + [InlineData(21, 12)] + [InlineData(49, 12)] + [InlineData(56, 12)] + [InlineData(21, 2)] + [InlineData(28, 2)] + public void Reads_a_cross_reference_stream_whose_length_row_a_spelled_endstream_parts_the_predicted_row_of(int columns, int predictor) { // Stream 7's /Length names object 5, which only its own row places, and its rows spell an endstream right after that - // row, which parts the second predicted row: the data up to it decodes, as data whose end the file gives, to the first - // row alone, the parted one dropped, and the row for the /Length was not found. The section was charged with the - // rows it lacked and the index rebuilt (#220). - var file = ChainFiles.DeferredLengthRowPartedBySpelledEndStream(columns); + // row, which parts the predicted row it lies in: the data up to it decodes, as data whose end the file gives, to the + // rows before that one, the parted one dropped — or, the first parted, to data the PNG predictor leaves as decoded, + // its tag bytes in the rows; or, under the TIFF predictor, to that row still differenced —, and the row for the + // /Length was not found, or found garbled. The section was charged with the rows it lacked and the index rebuilt + // (#220). + var file = ChainFiles.DeferredLengthRowPartedBySpelledEndStream(columns, predictor); using var document = PdfDocument.Open(file); @@ -3143,4 +3149,62 @@ public void Reads_a_cross_reference_stream_whose_length_row_a_spelled_endstream_ document.WasRepaired.Should().BeFalse(); document.GetObject(new PdfObjectId(3)).AsDictionary().Required().IsOfType(PdfName.Page).Should().BeTrue(); } + + [Theory] + [InlineData(0)] + [InlineData(3)] + [InlineData(6)] + public void Reads_a_cross_reference_stream_whose_rows_a_length_it_places_runs_past_spell_an_endstream_before_the_first(int prefix) + { + // Stream 7's /Length names object 5, which only its own row places, and its data spells an endstream before its first + // row: the data up to it holds no whole row, which the row for the /Length cannot lie in, nor a predicted row undone, + // and the section lacks every row, the file's. Asked to look for that row again, the reader would divide by the + // length of a row it took to be none. + var file = ChainFiles.DeferredLengthEndStreamSpelledBeforeRows(prefix); + + using var document = PdfDocument.Open(file); + + document.WasRepaired.Should().BeTrue(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.XRefRebuilt); + document.GetObject(new PdfObjectId(3)).AsDictionary().Required().IsOfType(PdfName.Page).Should().BeTrue(); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void A_decoding_guard_a_deferred_length_s_data_reaches_once_its_predicted_row_is_undone_is_reached_as_read_whole(bool rowBefore) + { + // Stream 7's /Length names object 5, which only its own row places, 1,820 bytes or more into the rows a RunLength step + // decodes after a predicted Flate one; its rows spell an endstream that parts a predicted row, past object 5's row or + // before it. MaxDecodedStreamLength, 1,100 bytes, stops the RunLength step read whole, the rows past it the guard's. + // The data up to the endstream, its end the file's, decodes within the bound; decoded again, its predicted row undone + // as far as it goes, to look for the /Length's row, it reached the bound, which the reader dropped with what that + // decoding reported: the section was charged with the rows it lacked and the index rebuilt, where ThrowOnLimit threw. + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxDecodedStreamLength = 1100 } }; + + using (var whole = PdfDocument.Open(ChainFiles.DeferredLengthRowPastRunLengthRuns(rowBefore, direct: true), options)) + { + whole.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitDecodedStream); + whole.WasRepaired.Should().BeFalse(); + } + + var file = ChainFiles.DeferredLengthRowPastRunLengthRuns(rowBefore); + + // Unbounded, the row before the endstream is found in the data decoded again, which is taken again to the length; one + // past it is not among the rows the data holds, which a length checked once the chain is read says (#220). + using (var unbounded = PdfDocument.Open(file)) + { + unbounded.WasRepaired.Should().Be(!rowBefore); + } + + using var document = PdfDocument.Open(file, options); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitDecodedStream); + document.WasRepaired.Should().BeFalse(); + document.Reader.Structure.Sections[0].CutByLimit.Should().BeTrue(); + document.GetObject(new PdfObjectId(3)).AsDictionary().Required().IsOfType(PdfName.Page).Should().BeTrue(); + + var throwing = () => PdfDocument.Open(file, options with { ThrowOnLimit = true }); + throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxDecodedStreamLength)); + } } From ee9c26e74a80ca07d0b3b361b41ab38bbc8f2497 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 07:18:30 +0000 Subject: [PATCH 139/146] test: hold the bound and the empty run of data a hidden length ends Two rules of the hidden end had no test that fails without them. Flate data whose deflate body ends in the data as taken is whole only where decoding stayed within MaxDecodedStreamLength: past it, the guard is reported and the data cut, as read whole. Data that is all white space and comments before its endstream decodes to nothing, cut by the guard, under LZW, Flate and RunLength. Refs #224 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 10450a70..29dea14b 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1439,6 +1439,60 @@ public void Flate_data_a_hidden_length_ends_where_its_deflate_body_does_decodes_ } } + [Theory] + [InlineData("cut", false)] + [InlineData("left out", true)] + public void A_decoding_guard_flate_data_a_hidden_length_ends_reaches_is_reached_as_read_whole(string guard, bool checksumWrong) + { + // Content stream 4's zlib data, its checksum right or wrong, under a /Length a guard cut or left out of the index, its + // endstream proven its own, decodes to 4 MB, past a MaxDecodedStreamLength of 1 MB, which stops it read whole. Taken + // for a deflate body that ends in the data, its output would be whole: the guard unreported, and what it cut the + // file's. + var zlib = Zlib(new byte[4 << 20]); + zlib[^1] ^= checksumWrong ? (byte)0x55 : (byte)0; + var file = HiddenLengthFile(guard, "/Filter /FlateDecode", zlib, "\n"); + var hidden = HiddenLengthOptions(guard); + + using (var whole = PdfDocument.Open(file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxDecodedStreamLength = 1 << 20 } })) + { + var read = whole.GetObject(new PdfObjectId(4)).AsStream().Required(); + PdfFilterPipeline.Decode(read, null, out _, out var cut).Length.Should().Be(1 << 20); + cut.Should().BeTrue(); + } + + using var guarded = PdfDocument.Open(file, hidden with { Limits = hidden.Limits with { MaxDecodedStreamLength = 1 << 20 } }); + var stream = guarded.GetObject(new PdfObjectId(4)).AsStream().Required(); + var decoded = PdfFilterPipeline.Decode(stream, null, out _, out var cutByGuard); + + stream.Data.EndHidden.Should().BeTrue(); + decoded.Length.Should().Be(1 << 20); + cutByGuard.Should().BeTrue(); + guarded.Diagnostics.Select(entry => entry.Code).Should().Equal(GuardCode(guard), PdfDiagnosticCodes.LimitDecodedStream); + } + + [Theory] + [InlineData("cut", "/LZWDecode", "\n\n")] + [InlineData("left out", "/FlateDecode", " \t\n")] + [InlineData("cut", "/RunLengthDecode", " % c\n\n")] + public void Data_that_is_all_the_run_a_hidden_length_may_leave_out_is_the_guard_s_to_lack_under_a_filter_with_an_end( + string guard, string filter, string tail) + { + // Content stream 4's data is nothing but white space, and a comment past it, before the endstream the next object + // proves its own, under a /Length a guard cut or left out of the index: the hidden length may leave it all out, and + // the LZW, Flate or RunLength data it gives, which a whole read ends where its format says, the guard's to lack. Under + // ASCIIHex, data of white space alone is whole (above). + var file = HiddenLengthFile(guard, "/Filter " + filter, [], tail); + + using var guarded = PdfDocument.Open(file, HiddenLengthOptions(guard)); + var stream = guarded.GetObject(new PdfObjectId(4)).AsStream().Required(); + var decoded = PdfFilterPipeline.Decode(stream, null, out _, out var cutByGuard); + + stream.Data.EndHidden.Should().BeTrue(); + decoded.Length.Should().Be(0); + cutByGuard.Should().BeTrue(); + guarded.Diagnostics.Select(entry => entry.Code).Should().Equal(GuardCode(guard)); + } + /// /// A file whose content stream 4 holds and before its endstream, which /// object 5 after it proves its own, its /Length object 20, which a MaxObjectLength of 1,024 cuts, or, left From 241d7014d38045bf5b9046e60571c93359832524 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 07:20:00 +0000 Subject: [PATCH 140/146] docs: say rows a section lacking rows may hold beside a cut are the guard's The architecture, ADR 34, the references, the concepts and the threat model said a section whose data lacks the rows its values declare beside a value a guard cut reads as read whole, the chain going on. They now say the guard is reached once the chain is read, and that rows the data may still hold, read with the cut value, are the guard's: the chain ends there and the validator says them unchecked. A deferred length's row is said to be found in any predicted row a spelled endstream parts, a decoding guard that search reaches standing. A comment cut short is completed, a missing checksum and a first number below 0 are named, and two tests are cited. Refs #220, #224, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- ...lid-pdf-is-readable-and-the-readers-guards-are.md | 3 ++- docs/architecture.md | 3 ++- docs/threat-model.md | 6 +++--- docs/website/docs/concepts/reader-limits.md | 3 ++- docs/website/docs/concepts/validation.md | 2 +- docs/website/docs/reference/diagnostics.md | 4 ++-- docs/website/docs/reference/reader-limits.md | 12 ++++++++---- src/AdCodicem.Pdf/IO/PdfFileReader.cs | 1 + 8 files changed, 21 insertions(+), 13 deletions(-) diff --git a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md index 0a7ba9f2..74f04815 100644 --- a/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md +++ b/docs/adr/0034-every-valid-pdf-is-readable-and-the-readers-guards-are.md @@ -127,7 +127,8 @@ data up to it then the guard's; a whole reference the cut left leads on, as read whole; a chain of filters ends there, its data left encoded; and a cross-reference section whose values lead to an object `MaxObjectLength` cut ends the chain, nothing rebuilt — unless what the values read say makes it unreadable whatever the cut hides, or data whose length the file gives, its filters written direct, lacks the rows they declare whatever it hides — the file's fault, such a section read as read whole, lacking rows -(amended on 2026-10-24) —, the older sections and their trailers the guard's to hide, and the guard reached once the chain +(amended on 2026-10-24), the chain going on where the data holds no row: rows it may hold, read with the cut value, are +the guard's (amended on 2026-10-25) —, the older sections and their trailers the guard's to hide, and the guard reached once the chain is read. So does a cross-reference stream's `/Length` the chain could not read that leads to such an object, wherever only that length could say how far the data runs: no `endstream` comes before the next object or the end of the file, or the rows spell one and none confirms what the cut left of the length, the rows past it then the guard's. A guard that diff --git a/docs/architecture.md b/docs/architecture.md index cc3a7d8e..2fcf86c4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -144,7 +144,8 @@ separate types, separate severity scales, and vocabularies that never share a co older sections and their trailers the guard's to hide, wherever near its named offset the section is found — unless what the values read say makes it unreadable whatever the cut hides, or data whose length the file gives, its filters written direct, lacks the rows they declare whatever it hides — the file's fault, such a section - read as read whole, lacking rows, its trailer merged and the chain going on. An + read as read whole, lacking rows, its trailer merged and the chain going on where the data holds no row at the + fewest bytes a row may take: rows it may hold, read with the cut value, are the guard's, the chain ending there. An object `MaxObjectLength` cut there is loaded once the chain is read, which reaches the guard — thrown at opening under `ThrowOnLimit` —, for a value the chain keeps: not for a place it dropped as no section. A cross-reference stream's `/Length` the chain could not read, which leads to such an object, ends the chain so wherever only it diff --git a/docs/threat-model.md b/docs/threat-model.md index d48ee2a7..8104fcab 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -283,8 +283,8 @@ Known gaps: | A classic table of any length | Its window grows ×4 from 64 KB up to `MaxXRefSectionLength`, and the rows read are kept. A table the guard cuts, or whose rows run past a full index, keeps its trailer, found past the rows left unread by a search a megabyte at a time (`ReadTrailerPastRows`): comments are skipped, and anything else no row holds is the section's fault, reported, the search going on to the `trailer` keyword, so that its `/Encrypt` still refuses the document. The search starts at the start of the line the cut fell in, outside any comment and past the table's own keyword; it reads each megabyte once, every byte of it looked at before the next is read, however many comments or stray bytes it holds, and is not made for a place a section may have been moved to, a guess. The chain ends there, since an older section's rows would stand for the objects the unread rows define. What those rows place reads as null until the guard is raised | `ReaderLimitsTests.A_table_whose_keywords_the_bound_cuts_reports_the_bound`, `A_table_its_length_bound_cuts_keeps_its_trailer_and_the_encryption_it_names`, `A_table_its_length_bound_cuts_ends_the_chain_though_its_trailer_names_an_older_section`, `What_ends_the_rows_a_length_bound_cut_is_judged_as_it_is_where_none_did`, `A_table_a_length_bound_cuts_that_runs_to_the_end_of_the_file_has_no_trailer`, `A_table_its_length_bound_cuts_keeps_the_encryption_its_trailer_names_past_what_no_row_holds`, `A_table_cut_before_its_keyword_ends_keeps_its_trailer`, `Rows_past_a_full_index_are_not_read_and_the_trailer_past_them_is`, `A_cut_in_a_comment_among_the_rows_searches_for_the_trailer_from_the_comment_s_start`, `The_trailer_past_rows_left_unread_is_found_wherever_the_search_s_window_ends`, `A_trailer_past_rows_left_unread_that_its_own_bound_cuts_keeps_what_precedes_the_cut`; `DocumentReaderTests.The_search_for_the_trailer_past_rows_left_unread_reads_the_file_once`; `CrossReferenceChainTests.Reads_an_object_a_length_bound_cut_out_of_its_table_as_null_until_the_guard_is_lifted`, `A_section_found_near_where_prev_names_it_is_not_searched_past_the_rows_a_guard_left_unread`; `WindowEdgeTests.A_classic_table_reads_whole_wherever_its_window_edge_falls_near_its_trailer` | No: seeds stay under 120 KB | | A subsection claiming a huge count, or numbering objects past `int.MaxValue` | The count is not negative, and the first number and the last, first + count − 1, compared by subtraction so that a count as large as a `long` cannot wrap it, are at most `int.MaxValue`; past either, the table is malformed. The rows are bounded by the window, the entries they give by `MaxIndexEntries`, and a count claiming more rows than the table holds makes it malformed where the rows end. A row giving an object in use a generation outside 0..65,535 is refused alone, and the rows after it read; no older section's row then stands for its object, which is the rebuild's to find. A free row is read whatever its generation, as producers give the free list's head 65,536 | `CrossReferenceRuleTests.A_subsection_header_without_a_first_number_and_a_count_it_can_hold_makes_the_table_malformed`, `A_subsection_numbering_rows_past_the_largest_object_number_makes_the_table_malformed`, `A_subsection_claiming_more_rows_than_the_table_holds_makes_it_malformed_where_the_rows_end`, `A_row_giving_an_object_in_use_a_generation_no_object_can_have_is_refused_alone`, `A_free_row_whose_generation_is_past_65535_is_read_in_silence`; `DocumentReaderTests.Serves_no_older_revision_of_an_object_whose_newest_row_is_refused` | Yes | | A trailer, or a cross-reference stream's dictionary, of any length | `MaxTrailerLength`, which bounds a cross-reference stream's dictionary alone, not its data ([#220]). The chain ends at a section the guard cut, or whose trailer it cut: an older section, an `/XRefStm` or a `/Root` past the cut is unknown, and what only older sections index reads as null until the guard is raised, rather than as the revision an update replaced — a section found a few bytes from where it was named too, its trailer's cut kept. A `/Root` a guard hid — past a trailer it cut, in an older trailer the chain stopped short of, at the section count or at a section whose rows a guard cut or with which the index filled — is unknown, no fault of the file: the reader looks for the catalog among the objects it indexed, rebuilds nothing when none is one, and `file.root-invalid` says nothing ([#175]); a `/Root` that names an object a guard left out of the index — at a section count, rows a guard cut or a full index, as the chain was read or as a rebuild filled the index —, through a chain of references too, is taken as read, null until the guard is lifted, and no other catalog is looked for, which could only be another object, one an update superseded or a stray one ([#175]); rows a guard cut that run on to no trailer, which the reader looks for to the file's end, lack one, which is the file's fault, and the reader rebuilds for it as a read of the whole table does; so does a trailer whose `/Prev` or `/XRefStm` is no offset in the file, or one the chain has read — at a section count, a full index or rows a guard cut alike —, and a cross-reference stream whose data a guard cut past its last row, the chain going on; a `/Prev`, or a table's `/XRefStm`, written as a reference, or a chain of them as far as a resolution follows one, is unknown where it leads to an object that runs past `MaxObjectLength` — under its own header, not another object's —, or to one whose number the index read so far lacks and whose row may lie among those the guard left unread: any row past the cut of a table; of a cross-reference stream, a row its `/Index` or `/Size` numbers from where the guard stopped it — the row the index had no room for, up to the last its data holds, or past the data a guard cut —. Such a `/Prev` or `/XRefStm` that leads to an object `MaxObjectLength` cut ends the chain there for the guard, as a cross-reference stream's `/W` does (below): nothing is rebuilt, the older sections and their trailers are the guard's to hide, and the guard is reached once the chain is read, reported or thrown at opening under `ThrowOnLimit`; a hybrid table's `/XRefStm` so hidden ends the chain with it, since the older sections' rows would stand for its stream's ([#263]). A hybrid file's stream's own `/Prev`, which the chain does not follow, going on through the table's, is not read at all, nor are the `/Prev` and `/XRefStm` of a table an `/XRefStm` names in a stream's place: neither names a section missing nor ends the chain for a guard, and the rule on trailer values judges them as written, a reference they hold not followed ([#263]). An object the guard cut after a whole reference it holds hides nothing: the chain reads through it, as a load does, and loads it once the chain is read. Elsewhere it is read as a whole read reads it: object 0, a number whose row was refused, a free row, another object's, or a row a stream's data decoded whole does not hold names no section in either read, which is reported. A section the guard cut is where it was named, and is not looked for near there: what lies near it would stand for it. An older cross-reference stream whose dictionary the guard cut before it said what it is — no `/Type` yet — is a section the guard cut, not one missing; an array, a dictionary of another type, or a stream whose dictionary was read whole is no section, cut or not | `ReaderLimitsTests.A_trailer_that_the_table_bound_cuts_is_read_through_a_window_of_its_own`; `HostileInputTests.Opening_a_chain_of_trailers_that_never_close_reads_a_bounded_amount`; `CrossReferenceChainTests.An_object_only_a_section_past_a_cut_trailer_indexes_is_unknown_rather_than_missing`, `A_cross_reference_stream_whose_dictionary_a_guard_cut_before_it_declared_itself_is_no_section_missing`, `A_trailer_a_guard_cut_past_its_prev_ends_the_chain`, `ReaderLimitsTests.A_section_a_prev_names_a_few_bytes_off_ends_the_chain_where_its_trailer_is_cut`, `A_root_a_trailer_cut_before_it_hides_is_not_judged_missing`, `A_root_a_guard_kept_the_trailers_from_giving_is_unknown_and_rebuilds_nothing`, `A_root_naming_an_object_a_guard_left_out_of_the_index_is_taken_as_read_and_no_other_is_looked_for`, `A_root_a_rebuild_that_fills_the_index_leaves_out_is_taken_as_read_and_no_other_is_looked_for`, `Rows_a_guard_cut_that_run_to_no_trailer_leave_the_root_the_file_lacks_to_be_found`, `A_trailer_past_a_guard_whose_older_sections_lie_nowhere_hides_no_root`, `A_cross_reference_stream_a_limit_cut_past_its_rows_hides_no_trailer`, `A_prev_written_as_a_reference_reads_past_a_guard_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_past_the_full_point_is_no_fault_of_the_file`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_no_row_past_it_places_reads_as_a_whole_read_reads_it`, `A_prev_a_stream_that_fills_the_index_writes_as_a_reference_to_an_object_past_the_object_guard_hides_the_trailers`, `A_prev_written_as_a_chain_of_references_past_a_guard_is_followed_as_far_as_a_whole_read_follows_it`, `A_prev_a_stream_a_guard_stopped_writes_as_a_reference_is_unknown_only_for_a_row_it_left_unread`, `A_table_that_fills_the_index_hides_the_trailer_of_the_hybrid_stream_its_xrefstm_names`; `CrossReferenceChainTests.A_cross_reference_stream_a_guard_cut_is_not_looked_for_near_where_it_was_named`, `A_stream_a_guard_cut_where_prev_names_it_is_not_found_again_in_the_table_that_names_it`, `An_object_a_guard_cut_where_prev_names_a_section_is_a_section_missing_when_it_cannot_be_one`; `GuardCutValueTests.A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s`; `GuardCutValueTests.A_prev_whose_chain_ends_at_its_last_object_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_section_whose_prev_a_guard_cut_keeps_the_chain_it_hides_when_it_is_found_where_it_was_moved`, `A_chain_a_guard_ends_at_a_hidden_prev_leaves_unjudged_what_needs_the_whole_chain`, `A_hybrid_stream_s_own_prev_is_not_followed_the_chain_going_on_through_the_table_s`, `A_table_an_xrefstm_names_in_a_stream_s_place_has_its_own_links_left_unread`, `A_prev_written_as_a_reference_to_a_real_is_judged_through_it_only_where_the_reader_follows_it` | Yes | -| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); so is one that declared itself a section before a guard cut it — a table's `xref` keyword, `/Type /XRef`. A place that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is a guess dropped, the next place tried; where the chain names a section, a guard that cut the dictionary of what declares another type cut no section — a CIDFont's, which read whole no stream data follows —, and relocation is tried, while one that cut a value of a stream there leaves it the section the guard cut, whatever type it gives, as read whole it is the section. What makes a section unreadable whatever a value a guard cut holds is the file's fault, as read whole, judged before the cut is taken for the guard's (`PdfFileReader.FaultBesideCut`): a value past the cut that no section read so far places, which the sections a whole read reads before make the file's to lack; a `/W` that gives no three widths, or none past 0; rows numbered wrongly — an `/Index` that holds no pairs, whatever its elements, a subsection out of range, or wrong by the element read whatever the cut one holds, a `/Size` with no `/Index` that gives no count —, as far as the values read, written direct or through references, say: a value a guard cut says nothing, nor does what depends on it, the `/Size` an `/Index` so cut may stand beside; and data whose length the file gives, its filters written direct, decoded whole, that holds fewer rows than the counts read declare however few bytes the widths read leave a row — a width past what an int counts taken as such —, which lacks them whatever the cut hides: the section is read as read whole, lacking rows, its trailer merged, the chain going on to its `/Prev` and the index rebuilt, where it was moved to too, though the rows it holds are not read | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_cross_reference_stream_whose_deferred_length_a_guard_cut_is_found_where_it_was_moved`, `A_cross_reference_stream_a_guard_cut_the_dictionary_of_after_its_type_is_the_section_where_it_was_moved`, `A_classic_table_a_guard_cut_the_rows_of_is_the_section_where_it_was_moved`, `A_stream_with_no_numbering_whose_width_a_guard_cut_is_no_section_the_guard_cut`, `A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild`, `What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_guard_cut_holds_is_the_file_s`, `A_place_a_section_may_have_moved_to_that_a_value_past_a_guard_s_cut_makes_unreadable_is_dropped`, `A_stream_whose_value_a_guard_cut_where_the_chain_names_a_section_is_the_section_the_guard_cut_whatever_its_type`, `A_cross_reference_stream_that_misspells_its_type_and_whose_value_a_guard_cut_is_the_section_the_guard_cut`, `A_section_a_value_s_cut_keeps_from_being_read_is_the_guard_s_where_nothing_read_says_it_is_wrong`, `A_section_whose_data_holds_fewer_rows_than_it_declares_whatever_a_value_a_guard_cut_holds_is_the_file_s_to_lack`, `Data_a_length_the_chain_cannot_read_yet_leaves_to_a_spelled_endstream_says_nothing_of_the_rows_it_holds` | No | -| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken, in a predicted row the spelled `endstream` parts too, the data decoded once more knowing its end hidden, that row undone as far as it goes and the reports dropped: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, may stay within the bound — the group counted against what the filters before the last share as well, and no more rows held than the first decoding read —: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_decoding_guard_deferred_data_reached_is_reported_once_whatever_room_the_diagnostics_have`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `Data_whose_end_a_guard_hides_leaves_the_filters_after_a_dropped_group_the_room_its_direct_twin_s_have`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file`, `A_cross_reference_stream_the_chain_read_proves_the_endstream_of_the_older_one_before_it`, `DocumentReaderTests.Reads_a_cross_reference_stream_whose_length_row_a_spelled_endstream_parts_the_predicted_row_of` | No | +| What the chain refers to — a cross-reference stream's dictionary written with references, an indirect `/Length`, a relocation candidate's, a trailer followed by `stream` — loading objects while the index is still being read, and a correction, a rebuild or a null cached then served afterward as what the file wrote | Nothing loads while the chain is read (`PdfFileReader.ReadWithoutLoading`): an object is read where a section already read places it, at the offset its row gives, under the reference's number and generation, within `MaxObjectLength` and `MaxNestedLoads`, and is not a stream; a reference to a reference is followed no further than `PdfReference.Resolve` follows it; nothing is corrected, rebuilt, cached, recorded or reported, and a guard that cuts it leaves it unread — but for a whole reference the cut left after the value, which is read through as a load reads it, the object then loaded once the chain is read, which reaches the guard, or throws it at opening under `ThrowOnLimit`. A value the rows need that cannot be read so makes the section malformed — but for one that leads to an object `MaxObjectLength` cut, a `/W`, `/Index`, `/Filter` or `/DecodeParms`, a `/Size` with no `/Index`, an element or a parameter of one, or a `/Length` whose data no `endstream` ends (below): the section is then the guard's, counted as cut by it, the chain ends there and nothing is rebuilt, and that object is loaded once the chain is read, which reports the guard, or throws it at opening under `ThrowOnLimit` ([#263], [#224]); a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain alike (above). A `/Length` that cannot be read so — no section read so far places it, or a guard cut it, at any link of its chain of references — waits for the chain: it is read and checked once the chain is, behind a copy of the index as the file wrote it, but for one a guard cut or left out of the index, which is not judged ([#224]). An object read at a place a section may have been moved to that is no section leaves nothing of its reading, nor what it read through; one read at the offset the chain names keeps all of its reading, as a load would — its reports, the guards it reached, what it read through —: an object read through a whole reference is loaded once the chain is read only for a value the chain keeps or what that offset holds — not for a place it guessed and dropped, nor for a `/Size` beside an `/Index`, which numbers no row. A cross-reference stream found near the offset it is named at, whose values lead to an object `MaxObjectLength` cut, is the section, the guard's there as where it is named exactly: the chain ends at it, nothing rebuilt ([#263]); so is one that declared itself a section before a guard cut it — a table's `xref` keyword, `/Type /XRef`. A place that declares another type — a CIDFont, whose `/W` gives widths —, or whose dictionary a guard cut before it said what it is, is a guess dropped, the next place tried; where the chain names a section, a guard that cut the dictionary of what declares another type cut no section — a CIDFont's, which read whole no stream data follows —, and relocation is tried, while one that cut a value of a stream there leaves it the section the guard cut, whatever type it gives, as read whole it is the section. What makes a section unreadable whatever a value a guard cut holds is the file's fault, as read whole, judged before the cut is taken for the guard's (`PdfFileReader.FaultBesideCut`): a value past the cut that no section read so far places, which the sections a whole read reads before make the file's to lack; a `/W` that gives no three widths, or none past 0; rows numbered wrongly — an `/Index` that holds no pairs, whatever its elements, a subsection out of range, or wrong by the element read whatever the cut one holds, a `/Size` with no `/Index` that gives no count —, as far as the values read, written direct or through references, say: a value a guard cut says nothing, nor does what depends on it, the `/Size` an `/Index` so cut may stand beside; and data whose length the file gives, its filters written direct, decoded whole, that holds fewer rows than the counts read declare however few bytes the widths read leave a row — a width past what an int counts taken as such —, which lacks them whatever the cut hides: the section is read as read whole, lacking rows, its trailer merged and the index rebuilt, where it was moved to too, and the object the guard cut loaded once the chain is read, which reaches the guard; the rows its data may hold at the fewest bytes a row may take, read with the cut value, are the guard's, the chain ending there, as where rows a guard cut, and the section said unchecked, its highest number not judged — the chain goes on to its `/Prev` only past data that holds none | `DocumentReaderTests.Changes_nothing_of_the_index_while_the_chain_is_read`, `A_value_the_chain_reads_that_a_guard_cuts_leaves_the_section_whole`, `Reads_a_value_the_chain_refers_to_past_the_window_it_starts_with`, `A_place_a_section_may_have_moved_to_that_holds_another_object_leaves_nothing_of_its_reading`, `A_place_a_section_may_have_moved_to_whose_dictionary_a_guard_cuts_is_no_section_the_chain_ends_at`, `A_cross_reference_stream_whose_dictionary_the_trailer_guard_cuts_before_it_declares_itself_reaches_the_guard`, `Reads_a_cross_reference_stream_whose_length_only_it_indexes_as_its_direct_twin`, `Checks_a_deferred_length_against_the_data_once_the_chain_is_read`, `Reads_every_row_of_a_cross_reference_stream_whose_data_a_deferred_length_leaves_to_its_end_of_line`, `A_guard_a_deferred_length_reaches_throws_once_the_chain_is_read_when_asked_to`, `A_deferred_length_whose_reference_leads_to_an_object_only_its_own_section_places_is_read_once_the_chain_is`; `CrossReferenceRuleTests.A_cross_reference_stream_value_the_chain_cannot_read_leaves_the_index_as_the_file_wrote_it`, `A_trailer_followed_by_stream_whose_length_the_chain_cannot_read_has_its_entry_judged_as_the_file_wrote_it`, `A_value_the_rows_need_that_the_chain_cannot_follow_to_its_end_makes_the_section_malformed`; `HostileInputTests.Opening_a_chain_whose_value_leads_through_lengths_longer_than_the_stack_is_deep_reads_a_bounded_amount`; `GuardCutValueTests.A_cross_reference_stream_value_a_guard_cut_ends_the_chain_as_the_guard_s`, `A_deferred_length_a_guard_cut_is_not_judged_once_the_chain_is_read`, `A_deferred_length_whose_reference_leads_to_an_object_a_guard_cut_is_not_judged_once_the_chain_is_read`; `DocumentReaderTests.A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to`, `A_deferred_length_s_fault_names_the_reference_at_the_end_of_its_chain`, `A_deferred_length_whose_chain_is_too_long_to_follow_is_reported_at_opening_as_a_later_load_reads_it`; `GuardCutValueTests.A_cross_reference_stream_s_length_whose_chain_ends_at_its_last_object_a_guard_cut_waits_for_the_chain`, `A_value_the_chain_reads_through_a_whole_reference_a_guard_cut_after_is_read_and_reaches_the_guard`, `An_object_read_through_for_a_place_that_holds_no_section_reaches_the_guard_only_where_a_kept_value_reads_it`, `What_the_offset_a_section_is_named_at_holds_keeps_what_it_read_through_as_a_load_would`, `A_size_beside_an_index_that_names_an_object_a_guard_cut_after_a_whole_reference_is_not_read`, `A_cross_reference_stream_a_guard_cut_a_value_of_is_found_where_it_was_moved`, `A_cross_reference_stream_whose_deferred_length_a_guard_cut_is_found_where_it_was_moved`, `A_cross_reference_stream_a_guard_cut_the_dictionary_of_after_its_type_is_the_section_where_it_was_moved`, `A_classic_table_a_guard_cut_the_rows_of_is_the_section_where_it_was_moved`, `A_stream_with_no_numbering_whose_width_a_guard_cut_is_no_section_the_guard_cut`, `A_font_whose_widths_a_guard_cut_where_the_chain_names_a_section_is_no_section_the_guard_cut`, `A_place_a_section_may_have_moved_to_that_cannot_be_read_leaves_what_it_read_through_to_the_rebuild`, `What_makes_a_cross_reference_stream_unreadable_whatever_a_value_a_guard_cut_holds_is_the_file_s`, `A_place_a_section_may_have_moved_to_that_a_value_past_a_guard_s_cut_makes_unreadable_is_dropped`, `A_stream_whose_value_a_guard_cut_where_the_chain_names_a_section_is_the_section_the_guard_cut_whatever_its_type`, `A_cross_reference_stream_that_misspells_its_type_and_whose_value_a_guard_cut_is_the_section_the_guard_cut`, `A_section_a_value_s_cut_keeps_from_being_read_is_the_guard_s_where_nothing_read_says_it_is_wrong`, `A_section_whose_data_holds_fewer_rows_than_it_declares_whatever_a_value_a_guard_cut_holds_is_the_file_s_to_lack`, `Rows_a_section_lacking_rows_holds_beside_a_value_a_guard_cut_are_the_guard_s_and_no_older_section_stands_for_them`, `A_section_lacking_rows_beside_a_value_a_guard_cut_reaches_the_guard_once_the_chain_is_read`, `The_validator_charges_the_file_with_no_size_rows_a_section_lacking_rows_holds_beside_a_value_a_guard_cut_hide`, `Data_a_length_the_chain_cannot_read_yet_leaves_to_a_spelled_endstream_says_nothing_of_the_rows_it_holds` | No | +| A cross-reference stream whose `/Length` the chain cannot read: its data searched through a window `MaxTrailerLength` bounds, ended early by rows that spell `endstream` or an object header, or, with no `endstream`, read to the end of the file as rows; a guard's cut of that `/Length` charged to the file | Its data runs to its first `endstream`, searched for once in the bytes the parser holds and then in the file past them, up to the next object the sections read so far place or the end of the file, as any stream's whose length is not confirmed (*Stream data and its length*), but stopped by no object header its rows spell and keeping a carriage return before the end-of-line as data, which a row may end in. With no `endstream` before that stop, or none searched for once the document's searches have read as much of the file as they may, the section is malformed, the file's fault and no guard's: "its /Length is the reference N G R, which no section read before it places where it can be read, and no endstream follows its data before …" — but where the `/Length` leads to an object `MaxObjectLength` cut, through a chain of references too, the section is the guard's, as one whose `/W` a guard cut is (above): "its /Length is the reference 5 0 R, which one of the reader's limits cut" ([#224]). Rows that spell an `endstream` are told from the data's end by the row for the `/Length` that reading the section's rows keeps — a row of a reserved type is ignored, as the specification says —, when it lies among the rows taken, in a predicted row the spelled `endstream` parts too — the row the data decoded as the file's drops, leaves as decoded, the first parted, or still differenced under the TIFF predictor —, the data decoded once more knowing its end hidden, that row undone as far as it goes and the reports dropped, but for a guard on decoding that decoding reached, which the whole data reaches too: short of the row, the rows past the bound are then the guard's, as where the decoding before reached it, and `ThrowOnLimit` throws as the report stands: the object it places is read there as the chain reads one, nothing kept, corrected or reported, and where an `endstream` confirms the length it gives, the data is taken again to it and decoded once more, what the first decoding reported dropped. Where `MaxObjectLength` cuts the object that row places — or the object a chain of references from it leads to, or the one a newer section places for the `/Length`, or one a chain from it leads to —, the integer read before the cut is only a place to look for `endstream` at, read once with the cut however many sections ask — a whole reference it holds is followed on, as read whole, to a length the file gave where it reaches an object read whole, rows the data taken again lacks then the file's —: the data is taken again where one confirms it — rows it still lacks then the guard's, the integer no length the file gave —, and otherwise the rows past the `endstream` the rows spell are the guard's, as rows past a decoding guard are — the section counts as cut, the chain ends there, nothing is charged to the file, and the length is not checked —; but where what follows the `endstream` the data ends at proves it the stream's own (*Stream data and its length*), the rows the data lacks are the file's, as read whole, and the index is rebuilt ([#224]). Otherwise, a row that does not stand — a newer section gave the number, or refused it; the row places it in an object stream, under another generation, or holds no length an `endstream` confirms — leaves the data as taken, and the length, checked once the chain is read, is reported where an `endstream` confirms it past the rows the chain read; data a guard cut that lacks rows — `MaxStreamLength` as it is read, or a guard on decoding it, `MaxDecodedStreamLength` or `MaxFilterChainLength`, which keeps it from being taken again — is the guard's, not checked against it; data that holds every row the section declares is checked whatever a guard did, but not against a length a guard cut. Wherever the guard hides where the data ends — a guard on decoding kept it, which lacks rows, from being taken again, or a guard's cut of the `/Length` gave its end —, it is decoded once more knowing that, as data a guard cut — a group ASCII85 or ASCIIHex holds in part, which the hidden end may have parted, dropped and counted against `MaxDecodedStreamLength` all the same, a predictor's last row undone as far as it goes, so that no row is read from bytes a guess completed —, a decoding guard the first decoding reached standing, with its report, though a filter after the one that drops a group, given less, may stay within the bound — the group counted against what the filters before the last share as well, and no more rows held than the first decoding read —: a filter run out at its end, over data a spelled `endstream` cut short, says nothing of the file, while a fault in what it read stands, as for the stream's direct twin, and so does a report the capacity counted ([#220]); data that holds every row the section declares is not decoded again, its end the file's | `GuardCutValueTests.A_decoding_guard_deferred_data_reached_stands_when_the_data_is_decoded_again_knowing_its_end_hidden`, `A_decoding_guard_deferred_data_reached_is_reported_once_whatever_room_the_diagnostics_have`, `A_group_an_unproven_endstream_parts_where_a_guard_cut_the_length_leaves_its_rows_to_the_guard`, `A_predicted_row_an_unproven_endstream_parts_where_a_guard_cut_the_length_is_undone_as_far_as_it_goes`; `DocumentReaderTests.A_cross_reference_stream_whose_length_only_it_indexes_is_read_past_the_first_window_as_its_direct_twin`, `A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length`, `A_deferred_length_s_data_is_read_once_more_than_its_direct_twin_s`, `Rows_that_spell_an_object_header_do_not_end_a_deferred_length_s_data`, `A_deferred_length_s_data_with_no_endstream_before_the_end_of_the_file_makes_its_section_malformed`, `A_deferred_length_s_data_with_no_endstream_before_the_next_object_makes_its_section_malformed`, `Rows_that_spell_endstream_after_the_length_s_own_row_are_read_as_the_length_gives`, `Rows_that_spell_endstream_in_an_update_leave_none_of_its_objects_to_the_revision_it_supersedes`, `Rows_that_spell_endstream_in_an_older_section_are_read_as_its_length_gives`, `A_length_row_that_does_not_stand_leaves_the_data_up_to_the_endstream_the_rows_spell`, `Updates_each_with_a_long_deferred_length_are_all_searched`, `A_deferred_length_is_taken_again_by_the_row_the_section_keeps_for_it`, `A_deferred_length_is_not_checked_against_data_a_guard_cut`, `A_filter_guard_that_keeps_a_deferred_length_s_data_from_being_taken_again_leaves_no_filter_fault`, `A_guard_s_report_the_capacity_drops_beside_a_deferred_length_s_decoding_is_counted`, `A_fault_a_deferred_length_s_data_holds_before_a_guard_stops_it_is_the_file_s_as_for_a_direct_one`, `A_deferred_length_s_data_whose_last_digit_alone_crosses_MaxDecodedStreamLength_is_cut_as_its_direct_twin_s`, `Data_whose_end_a_guard_hides_leaves_the_filters_after_a_dropped_group_the_room_its_direct_twin_s_have`, `A_deferred_length_s_data_whose_rows_a_guard_on_decoding_holds_all_of_reports_its_filters_ends`; `GuardCutValueTests.A_deferred_length_a_guard_cut_leaves_a_stream_with_no_endstream_the_guard_s`, `The_integer_a_guard_cut_a_deferred_length_after_is_where_rows_that_spell_an_endstream_are_taken_again_to`, `Rows_past_an_endstream_they_spell_are_the_guard_s_when_it_cut_the_deferred_length_before_its_value`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_leads_to_an_object_a_newer_section_places_and_it_cut`, `The_integer_a_guard_cut_a_length_a_newer_section_places_after_is_where_spelled_rows_are_taken_again_to`, `Rows_data_taken_again_to_an_integer_a_guard_parted_still_lacks_are_the_guard_s`, `Rows_data_taken_again_to_an_integer_a_guard_parted_in_the_own_row_s_object_still_lacks_are_the_guard_s`, `A_whole_reference_a_guard_cut_after_leads_the_own_row_s_length_on_as_a_whole_read_follows_it`, `A_whole_reference_a_guard_cut_after_leads_a_chain_on_whichever_section_reads_it_first`, `A_filter_run_out_where_a_guard_s_cut_of_the_length_leaves_the_data_says_nothing_of_the_file`; `ReaderLimitsTests.MaxTrailerLength_bounds_a_cross_reference_stream_s_dictionary_and_not_its_data`; `HostileInputTests.A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times`, `Sections_whose_lengths_name_one_object_a_guard_cut_read_it_once`; `DocumentReaderTests.A_deferred_length_a_newer_section_refused_takes_no_data_again`, `A_deferred_length_is_taken_again_by_a_row_with_no_type_field`; `GuardCutValueTests.A_cross_reference_stream_with_no_endstream_whose_length_s_chain_ends_at_its_last_object_a_guard_cut_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_file_s_when_the_length_s_chain_gives_a_wrong_length`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`, `A_cross_reference_stream_s_length_a_guard_cut_leaves_rows_its_proven_endstream_ends_to_the_file`, `A_cross_reference_stream_the_chain_read_proves_the_endstream_of_the_older_one_before_it`, `GuardCutValueTests.Rows_a_predicted_row_undone_as_far_as_it_goes_completes_are_read_though_no_guard_stopped_the_decoding_before`; `DocumentReaderTests.Reads_a_cross_reference_stream_whose_length_row_a_spelled_endstream_parts_the_predicted_row_of`, `A_decoding_guard_a_deferred_length_s_data_reaches_once_its_predicted_row_is_undone_is_reached_as_read_whole`, `Reads_a_cross_reference_stream_whose_rows_a_length_it_places_runs_past_spell_an_endstream_before_the_first` | No | | `/W` widths that overflow a field or give empty rows; `/Size` or `/Index` claiming rows the data lacks, or numbering objects no object can be; row fields an entry cannot hold | At least three widths, each an integer that is not negative, not all zero, narrowed only once it is known to be within the decoded data: a row longer than the data is one the data does not hold, and the widths' sum is then within a `long`. A field wider than 8 bytes is read when its leading bytes are zero, its every byte still read. `/Index` is an array of pairs of integers, each numbering its rows within 0..`int.MaxValue`, compared by subtraction, with a count that is not negative, each read once; without `/Index`, `/Size` is a non-negative integer. A fault in this numbering makes the section malformed before any row is read. Rows are read only while they fit in the decoded data, and the shortfall is computed by division, where a `/Size` as large as a `long` cannot overflow. Each field is read as an unsigned 64-bit value, one wider than 8 bytes whose leading bytes are not all zero as the largest; a row is refused alone when no entry can hold it — an offset of 2^63 or more, a generation past 65,535, an object stream numbered outside 1..`int.MaxValue`, an index past `int.MaxValue`, no older section's row then standing for its object — and a type past 2 is ignored, as ISO 32000-1 asks | `CrossReferenceRuleTests.A_cross_reference_stream_whose_widths_or_ranges_cannot_be_read_is_malformed`, `A_cross_reference_stream_whose_size_declares_more_rows_than_it_holds_is_malformed`, `A_cross_reference_stream_row_whose_fields_no_entry_can_hold_is_refused_alone`, `A_cross_reference_stream_row_of_a_type_past_2_is_ignored_however_wide_its_type_field`, `A_cross_reference_stream_whose_fields_are_wider_than_8_bytes_is_read_when_their_leading_bytes_are_zero`, `A_cross_reference_stream_row_whose_wide_field_holds_more_than_64_bits_is_refused_alone`, `A_cross_reference_stream_whose_rows_are_wider_than_its_data_holds_fewer_rows_than_it_declares`; `HostileInputTests.Survives_a_cross_reference_stream_with_impossible_field_widths`; `CrossReferenceChainTests.Looks_for_an_object_a_cross_reference_stream_holds_no_row_for` | Seldom: the rows are compressed | | Entries without end — rows of one byte decoding to 256 MB from a few hundred bytes, refused rows, a rebuild's headers, the members of the object streams a rebuild expands | `MaxIndexEntries` counts the distinct object numbers the index holds, in use, free or refused: a row or a header for a number held already takes no room, so that millions of copies of one header hide nothing behind them, and a rebuild reads the file to its end. The first number with no room is reported once, where it was met; the chain stops at the section that filled the index, and a stream's rows past it are not read ([#164], [#183]) | `HostileInputTests.A_cross_reference_stream_of_one_byte_rows_decoding_to_the_bound_indexes_no_more_entries_than_the_guard_allows`, `Rows_a_section_refuses_count_toward_the_entries_the_index_may_hold`, `A_rebuild_finds_a_document_behind_millions_of_repeated_headers_of_one_object`; `CrossReferenceChainTests.The_entry_guard_counts_the_entries_of_every_section_and_stops_the_chain_at_the_one_that_fills_it`; `ReaderLimitsTests.A_rebuild_indexes_object_numbers_up_to_the_entry_guard_and_reports_the_first_it_had_no_room_for`, `A_rebuild_past_the_entry_guard_still_takes_the_newest_definition_of_a_number_it_holds`, `A_rebuild_takes_in_the_members_of_object_streams_no_further_than_the_entry_guard`, `A_rebuild_that_reaches_the_entry_guard_finishes_the_index_before_it_throws`, `A_row_an_older_section_refuses_for_a_number_a_newer_one_gave_takes_no_room` | Yes | | A declared count the index makes room for before its rows are read | A section makes room for its rows before they are read, so that the map is sized once: no more rows than a stream's data holds, or a classic table's bytes hold — a line each, of 19 bytes at least — up to the first byte no row holds, read in the window that holds that byte or more rows than the index has room for, and made once however often the rows are read; no more numbers than `MaxIndexEntries` lets the index take, nor than the newest section's `/Size` numbers, written direct — an integer, or a real with no fractional part; one written as a reference is not read through for it, and bounds nothing —, or the highest number the sections read so far give when it is stale, an older section's superseded rows adding none. A `/Size` alone makes no room. Room is never added by less than the map holds, so that a chain of many sections does not resize it at each one; once a doubling comes within a doubling of the newest `/Size`, room is made for that size, no more than four times what the map held. Rows that index nothing — a reserved type, a refused row, a number held already — still get room, at most 52 bytes a number the guard lets the index take, once a document | `HostileInputTests.A_declared_size_far_past_the_rows_a_file_holds_makes_room_for_no_more_than_they_hold`, `Rows_that_index_nothing_make_room_for_no_more_entries_than_the_guard_allows`, `Blank_lines_among_a_table_s_rows_make_room_for_no_more_rows_than_their_bytes_hold`; `ReaderLimitsTests.A_table_its_length_guard_cuts_makes_room_for_no_more_rows_than_the_guard_lets_the_reader_read`, `A_newest_size_makes_room_for_no_object_the_sections_hold_no_row_for`; `XRefTableTests.Room_for_a_few_rows_more_than_the_table_holds_doubles_it_rather_than_growing_it_by_them`, `Room_is_made_for_no_more_entries_than_the_most_asked`; `DocumentReaderTests.Opening_a_chain_of_a_thousand_sections_resizes_the_index_no_more_often_than_doubling_it`, `An_update_s_rows_and_the_original_s_make_room_for_the_objects_the_newest_size_numbers`, `A_table_of_many_subsections_makes_room_for_its_rows_once`, `A_subsection_claiming_millions_of_rows_makes_room_for_the_rows_its_bytes_hold`, `A_table_read_again_in_a_larger_window_makes_room_for_its_rows_once`, `Rows_written_a_byte_short_make_room_for_each_of_them`, `A_newest_size_an_update_did_not_raise_makes_room_for_the_objects_its_rows_add`, `A_lowered_entry_guard_reads_a_table_in_no_larger_window_than_its_room_needs` | Yes | @@ -358,7 +358,7 @@ Known gaps: | A stream's data read through the window its object is parsed through: a wrong `/Length` growing it to `MaxObjectLength` for a few bytes of data, and the object then reported cut | A stream's data never counts toward `MaxObjectLength`, however its end is found, nor the end-of-line after its `stream` keyword: no window is grown to find an `endstream`, nor where the data starts, which the file says past a window a guard ended on the keyword; the data past the bytes the parser holds is searched in the file, and a fault of its length is the file's, not the guard's ([#174]) | `StreamLengthTests.A_stream_s_data_counts_for_nothing_toward_MaxObjectLength_and_its_fault_is_the_file_s`, `A_stream_whose_keyword_a_limit_cuts_records_nothing_of_its_length`; `WindowEdgeTests.A_stream_reads_the_same_when_a_guard_s_window_ends_on_its_keyword`, `A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword`; `HostileInputTests.A_stream_whose_endobj_follows_its_length_without_endstream_reads_no_further_than_its_window_whatever_follows`; `WindowEdgeTests.A_keyword_longer_than_stream_reads_the_same_wherever_the_window_edge_cuts_it`; `ParserTests.Leaves_a_stream_whose_window_ends_on_its_keyword_to_a_larger_one`, `Leaves_a_stream_with_no_endstream_in_a_window_to_a_larger_one` | No | | Confirming a length by reading past the buffer or the file, or through white space without end; a `%` of the data read as a comment that confirms a length too short | An `endstream` after the declared length confirms it, after any white space and comments, however many ([#174]); the 35,716 streams of the corpus whose length is confirmed have a gap of 0 to 2 bytes. A comment counts there only once white space comes before it: a `%` right where the length ends is the data's, so the length goes unconfirmed and the search below finds where the data ends. The bytes the parser holds decide when they can, and what follows an `endstream` they hold, when they end before it shows, is read on in the file, as past them: where the window ends decides nothing of the object's `endobj`. Past them, a first read of 64 bytes on the stack (`PdfObjectParser.EndObjLookahead`, a first read and no bound), refused outside the file and clamped to it, then reads that double up to 64 KB as far as the white space and comments run, once for each place, counted with the searches below; once those have read as much as they may, the first read alone decides. A length past the file confirms nothing | `WindowEdgeTests.A_stream_the_file_cuts_short_keeps_its_declared_length_without_asking_the_source_past_its_end`, `What_follows_an_endstream_the_window_holds_is_read_wherever_the_window_edge_falls`; `StreamLengthTests.A_stream_past_the_window_whose_length_endstream_follows_is_read_as_declared_for_a_few_bytes_more`, `A_length_any_white_space_or_comment_separates_from_endstream_is_confirmed`, `White_space_before_endstream_that_runs_past_the_first_read_is_read_on_and_confirms_the_length`, `What_follows_the_endstream_of_a_stream_past_the_window_is_seen_however_far_white_space_runs`, `A_percent_sign_where_a_short_length_ends_is_data_and_no_comment_that_confirms_the_length`, `A_comment_after_white_space_where_the_length_ends_confirms_it`; `ParserTests.Reads_endstream_after_any_white_space_and_comments_or_says_the_bytes_cannot_tell`, `Bytes_that_end_the_file_before_endstream_hold_none`; `StreamLengthTests.A_comment_in_the_gap_that_runs_past_the_window_is_read_on_and_its_check_kept_for_a_second_reading`, `An_endstreamx_read_on_past_the_window_confirms_the_length_and_is_no_endobj`, `White_space_after_a_length_past_the_window_is_read_on_to_the_end_of_the_file` | Property | | A search for a lost `endstream` running into other objects, or past its own object's end | A length unconfirmed, or none to confirm, sends the stream to one search for its first `endstream`: in the bytes the parser holds, then in the file, up to the nearer of the first object header the bytes hold (`HeaderText`) and the next object the index as written places, or to the end of the data. A stream whose `/Length` leads to an object a guard cut or left out of the index may be valid, its length only hidden, and its data may spell a header, as an embedded file's does — and the `endstream` of a stream the embedded file holds: a header the data spells that stops the search, no `endobj` of the object's own before it, leaves the data up to it to the guard, cut by it, the object's `endobj` unjudged, nothing said of the file; so does an `endstream` the data spells — a string's, say, an `endobj` after it or not —, which only the hidden length could tell from the stream's own. An `endstream` ends data that is the file's only where what follows the object's end — its `endobj`, or the `endstream` where none follows — is the next object the index places — at its row, or past the white space a row names before its header —, a cross-reference stream the chain read, which no row of its own need place, or the end of the file, which data that goes on is never followed by, or a whole `xref` or `startxref` keyword, which data may spell (below): read once for each place, within what the searches may read (`IPdfStreamDataProvider.IsOwnEndStream`). Proven, it ends data that is the file's, its `endobj` judged; but white space before the keyword — an end-of-line the search leaves out, or white space and comments the data ends in — only the hidden length says is the data's or not: the data's end is decoded as one the guard hides (`PdfStreamData.EndHidden`). The first filter decodes the data up to where that run starts, found in time linear in the data — a comment past white space within the data, never at its first byte, and none in ASCII85 data, where a `%` is a digit —: what it meets before is the file's, its end marker counting only there, its running out there — a Flate checksum short, an end marker missing — says nothing of the file, and where its output may lack what the run holds — Flate that lost its tail, LZW or RunLength data without its end marker — the output's end is the guard's, cut, for the predictor, which runs on Flate and LZW steps only and undoes the last row as far as it goes, the filters after it and an object stream's last member. Flate data whose deflate body ends in the data as taken decodes whole, its repair reported and a checksum that disagrees only where it reaches into the run silenced; white space decodes to nothing in ASCII85 or ASCIIHex data, whose output is whole, and so does a comment in ASCIIHex, which no valid hexadecimal data holds; the filters after a first whose output is whole decode as read whole, their faults the file's. A reading that cannot tell where the object ends takes back what an earlier one recorded of its `endobj` ([#224]). With none before that stop, the object's `endobj` plays its part ([#174]). A declared length that ends before the stop and that its `endobj` follows, past white space and comments as above, is kept, `stream.truncated`. Otherwise the object's own `endobj`, with only white space and comments between it and the stop, ends the data, its end-of-line left out, `stream.truncated`; an `endobj` a token follows past its comments ends nothing. That `endobj` is followed forward through the search's own reads, which read every byte up to the stop already (`EndObjTail`): no byte is read again to find it. Otherwise a declared length that ends before the stop is kept, `stream.length-invalid`, and its `endobj` is not judged. Otherwise the data runs to the stop, `stream.truncated` at the end of the file and `stream.length-invalid` at the next object, and the object lacks its `endobj`, unless the data taken holds one | `StreamLengthTests.A_stream_past_the_window_with_no_endstream_before_the_next_object_ends_at_its_object_s_endobj`, `A_stream_past_the_window_with_no_endstream_before_an_object_the_window_holds_ends_at_its_endobj_unsearched`, `The_search_stops_at_the_next_object_of_an_index_rebuilt_as_the_document_opened`, `A_stream_that_lost_its_endstream_takes_nothing_of_the_object_after_it`, `A_stream_that_lost_its_endstream_runs_to_a_header_its_object_has_no_endobj_before_and_lacks_its_endobj`, `A_stream_that_endobj_follows_without_its_endstream_keeps_its_declared_length`, `A_stream_whose_endobj_the_trailer_follows_runs_to_the_end_of_the_file_and_leaves_its_endobj_unjudged`, `IPRES_s_stream_that_lost_its_endstream_keeps_the_length_its_endobj_follows`, `A_percent_sign_where_a_short_length_ends_is_data_before_the_object_s_endobj_too`, `An_object_s_endobj_ends_data_with_no_endstream_whatever_comments_follow_it`, `An_endobj_a_token_follows_past_its_comments_ends_no_data`, `A_file_that_ends_on_the_stream_keyword_ends_the_stream_s_data_there_and_says_so`; `WindowEdgeTests.An_object_s_endobj_ends_data_with_no_endstream_wherever_the_reads_part_what_follows_it`; `ParserTests.A_stream_that_lost_its_endstream_ends_at_its_own_endobj_before_the_next_object`, `Ends_a_stream_s_data_at_its_object_s_endobj_that_only_white_space_and_comments_follow`, `A_stream_that_lost_its_endstream_runs_to_the_header_its_bytes_hold_when_no_endobj_precedes_it`, `A_window_given_to_the_parser_alone_that_cannot_say_where_a_stream_ends_asks_for_more`; `GuardCutValueTests.Data_a_header_stops_where_a_guard_hid_the_length_is_the_guard_s`, `A_stream_a_guard_hid_the_length_of_ends_at_its_own_endobj_before_a_header`, `An_endstream_the_data_spells_where_a_guard_hid_the_length_ends_data_that_is_the_guard_s`, `An_endstream_the_next_object_follows_where_a_guard_hid_the_length_lacks_its_endobj_as_the_file_s_fault`, `An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file_s_where_what_follows_proves_it_its_own`, `A_cross_reference_stream_with_no_row_of_its_own_proves_the_endstream_before_it`, `A_cross_reference_stream_named_before_white_space_proves_the_endstream_before_it_however_much`, `Flate_data_whose_last_byte_is_a_carriage_return_before_a_proven_endstream_is_no_fault_of_the_file`, `Data_that_ends_in_white_space_right_before_a_proven_endstream_ends_where_the_guard_hides`, `A_checksum_short_of_a_byte_white_space_before_a_proven_endstream_completes_is_no_fault_of_the_file`, `A_checksum_that_disagrees_before_the_white_space_a_hidden_length_leaves_is_the_file_s`, `A_filter_after_one_that_ends_where_its_format_says_decodes_as_read_whole_whatever_hides_the_end`, `Predicted_rows_a_hidden_end_leaves_a_byte_short_are_undone_as_far_as_they_go`, `Lzw_rows_a_hidden_end_leaves_a_byte_short_are_undone_as_far_as_they_go`, `Lzw_data_that_lacks_its_end_code_before_a_proven_endstream_past_an_end_of_line_is_no_fault_of_the_file`, `White_space_and_comments_a_hidden_length_leaves_are_not_decoded_as_data_that_lacks_its_end_marker`, `An_object_stream_whose_last_code_a_hidden_end_leaves_a_byte_short_ends_where_the_guard_hides`, `Data_a_hidden_length_ends_decodes_as_read_whole_but_for_what_the_run_after_it_may_hold`, `Flate_data_a_hidden_length_ends_where_its_deflate_body_does_decodes_as_read_whole`, `An_ascii85_object_stream_that_lacks_its_end_marker_before_a_proven_endstream_decodes_as_read_whole`, `An_endstream_and_an_endobj_the_data_spells_where_a_guard_hid_the_length_end_data_that_is_the_guard_s`, `Rows_past_a_spelled_endstream_are_the_guard_s_when_the_length_s_chain_ends_at_its_last_object_a_guard_cut`; `StreamLengthTests.A_stream_left_unsearched_is_reported_once_however_often_it_is_parsed`, `A_stream_left_unsearched_whose_length_runs_past_the_file_takes_the_data_its_object_holds`, `A_stream_read_once_the_searches_read_as_much_as_they_may_is_not_read_on_past_its_window`; `WindowEdgeTests.An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge_cuts_it`, `An_endobj_that_ends_a_longer_token_ends_no_data_wherever_the_reads_part_it` | Plausibly | -| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file, three times over the white space a cross-reference stream's search takes as data that its length, once read, is read on through, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, and the white space it crosses once more at most — the run before an `endobj`, or before what follows the keyword where no `endobj` does, taken from the read-on that read it, a run a row names before the next header read once from the row —, about three times the file in all; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `DocumentReaderTests.Deferred_lengths_followed_by_long_white_space_have_it_read_on_once_however_their_data_ends`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_proofs_of_streams_white_space_follows_read_less_than_three_times_the_file`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | +| Many searches, and reads on through white space, over one stretch of the file | What the searches read past the bytes the parser holds, and the reads on through white space and comments after a declared length, add up to at most four times the file in a document (`EndStreamSearchPasses`, an internal constant, with its reason): a valid file is read past the window only by a cross-reference stream whose `/Length` the chain cannot read, searched once, by a stream whose `/Length` a guard cut or left out of the index, searched once, and by white space after a declared length that runs past a first read, about twice the file, three times over the white space a cross-reference stream's search takes as data that its length, once read, is read on through, and the proof that an `endstream` found under a hidden `/Length` is the stream's own, 48 or 64 bytes past each, once, and the white space it crosses once more at most — the run before an `endobj`, or before what follows the keyword where no `endobj` does, taken from the read-on that read it, a run a row names before the next header read once from the row —, about three times the file in all; any other search is of an invalid stream, and reads about twice what its object spans. Past the bound, only the bytes the parser holds are searched: a declared length that ends before the next object the index places, or the end of the file, is kept, and the data otherwise runs to it, each reported as taken without a search — but where a guard hid the `/Length`, which would say how far the data runs, the data up to the stop is the guard's, its object's `endobj` unjudged, nothing reported ([#224]) | `StreamLengthTests.Streams_whose_data_share_a_stretch_read_the_file_a_bounded_number_of_times_over`; `DocumentReaderTests.Deferred_lengths_followed_by_long_white_space_have_it_read_on_once_however_their_data_ends`; `GuardCutValueTests.The_proofs_of_small_streams_whose_length_a_guard_hid_read_less_than_the_file`, `The_proofs_of_streams_white_space_follows_read_less_than_three_times_the_file`, `The_proofs_of_endstreams_data_spells_before_white_space_read_less_than_three_times_the_file`, `The_place_a_row_names_before_the_next_header_is_read_once_however_often_the_stream_is_parsed`; `StreamLengthTests.A_stream_left_unsearched_whose_length_a_guard_hid_is_the_guard_s`; `HostileInputTests.Many_lengths_the_file_cannot_hold_read_the_file_once_not_once_each`, `A_deferred_length_s_data_with_no_endstream_makes_its_section_malformed_reading_the_file_a_few_times` | No | | One search reading without end | Reads that double from twice what the parser holds of the data up to 64 KB, each starting further on than the last, ending at the next object the index places, at the end of the file, at a header, at an `endstream`, or on an empty read | `StreamLengthTests.A_file_of_data_with_no_endstream_is_searched_once_to_its_end`, `A_source_that_returns_less_than_it_holds_ends_the_search` | No | | The same search made at every new parse | The result is kept by data start for the life of the document, and so is what the white space after a declared length leads to. Whether the object's `endobj` follows the declared length is read again only when that length changes, as a rebuild that replaces the `/Length` changes it | `StreamLengthTests.A_stream_whose_length_is_wrong_is_searched_and_reported_once_however_often_the_cache_lets_it_go`; `ReadOrderTests.A_stream_read_again_after_a_rebuild_changed_its_length_ends_as_one_read_after_the_rebuild_does` | — | | Encoded data of any length: a `/Length` of a few digits claiming gigabytes, one past what an array holds, which would exhaust memory at each read, or data decoding hands back as it is, which `MaxDecodedStreamLength` does not bound | `MaxStreamLength` (*The guards*): what is read of one stream's data at once stops at it, what lies past it is cut by a guard and reported as `limit.stream-length` — once for the stream in its document's diagnostics, and with each decoding that meets the cut in diagnostics a caller passes to `Decode`, as `limit.decoded-stream` is —, and the filters decode what was read as data the reader cut ([#170]) | `ReaderLimitsTests.A_stream_cut_at_MaxStreamLength_decodes_what_lies_within_it_as_data_the_reader_cut_and_is_reported_once`, `Data_decoding_hands_back_as_it_is_is_bounded_by_MaxStreamLength_and_not_by_MaxDecodedStreamLength`, `A_length_past_what_an_array_holds_ends_at_the_guard_and_never_in_an_exhausted_memory`, `Decoding_into_diagnostics_of_one_s_own_reports_the_cut_max_stream_length_made_each_time` | No | diff --git a/docs/website/docs/concepts/reader-limits.md b/docs/website/docs/concepts/reader-limits.md index f64c9620..867a4922 100644 --- a/docs/website/docs/concepts/reader-limits.md +++ b/docs/website/docs/concepts/reader-limits.md @@ -66,7 +66,8 @@ follows that is so named, or a cross-reference stream whose values lead to an ob section a guard cut does, wherever near its named offset the section is found — unless what the values read say makes the section unreadable whatever the cut hides, or data whose length the file gives, its filters written direct, lacks the rows they declare whatever it hides: a fault of the file, as read whole, such a section read as read whole, lacking -rows. +rows. The rows such data may still hold are read with the value the guard cut, and are the guard's: the chain ends at +that section, as where a guard cut its rows, since an older section's rows would stand for them. What a guard leaves out of the cross-reference index — the oldest sections of a long chain, the rows past a long table's bound, the objects past the most the index holds — is no damage either. Those objects read as null, unknown diff --git a/docs/website/docs/concepts/validation.md b/docs/website/docs/concepts/validation.md index f99928f5..0a06c88f 100644 --- a/docs/website/docs/concepts/validation.md +++ b/docs/website/docs/concepts/validation.md @@ -159,7 +159,7 @@ limit lets it check the rest. The same holds of a value that leads to such an ob malformed — so is one whose `/Length`, which the chain could not read, leads to one where only that length could say where its data ends —, unless what the values read say makes it unreadable whatever the guard hid, or data whose length the file gives, its filters written direct, lacks the rows they declare whatever it hid — the file's, such a section read as read whole, -lacking rows; +lacking rows, the rows it may still hold said not checked, nor the `/Size` they bear on; a `/Prev` or an `/XRefStm` the reader follows that leads to one ends the chain there rather than naming a section not found; and an object stream whose `/Filter` or `/DecodeParms` leads to one, or to an object a limit left out of the index, is left encoded, the objects it holds said not checked rather than lost diff --git a/docs/website/docs/reference/diagnostics.md b/docs/website/docs/reference/diagnostics.md index 64e075da..96c74df6 100644 --- a/docs/website/docs/reference/diagnostics.md +++ b/docs/website/docs/reference/diagnostics.md @@ -105,7 +105,7 @@ Codes are stable: they are part of the public contract, because callers filter o | `xref.section-unreadable` | A cross-reference section `/Prev` or `/XRefStm` names is there and cannot be read — a stray token among a table's rows or in place of its trailer, a subsection numbering objects past 2,147,483,647, a stream whose `/W`, `/Index`, data or — with no `/Index` — `/Size` cannot give rows, or whose `/W`, `/Index`, `/Filter` or `/DecodeParms`, or `/Size` with no `/Index`, is a reference no section read before it places where it can be read without loading it, or whose `/Length` is such a reference, or leads to one through others, and no `endstream` follows its data before the next object the sections read so far place, or before the end of the file, or none was looked for once the document's searches for `endstream` read as much of the file as they may — and nothing nearby can be read in its place; the rows read before the fault are kept, and what only the rest indexed is found by rebuilding the index when it is asked for. Reported at the section, the fault in the message. A value that names an object one of the reader's limits cut, directly or through a chain of references — a `/Length` whose data no `endstream` ends before that stop among them —, is the guard's, not the file's: the section is not reported, nor looked for nearby, the chain ends there, nothing is rebuilt, what only that section and the older ones index reads as null, and the guard is reported once the chain is read — but for what makes the section unreadable whatever the value holds, which is the file's and reported here, as read whole: a value past it no section read so far places, a `/W` that gives no three widths or none past 0, or rows numbered wrongly as far as the values read say — a subsection wrong by the element read whatever the cut one holds among them —; data whose length the file gives, its filters written direct, decoded whole, that holds fewer rows than the counts read declare however few bytes the widths read leave a row is read as read whole instead, lacking rows, its trailer merged and the chain going on, where it was moved to too | | `xref.chain-cycle` | The chain of previous sections looped: a `/Prev` or `/XRefStm` names an offset the chain has already named, whether it read a section there or not. Reported at that offset, or, when it lies outside the file, at the section whose trailer named it | | `xref.entry-out-of-range` | An object's entry places it outside the file — past its end, before its start, or past what a `long` holds once the header's offset is added —; reported with no position, the offset in the message, exactly. A section named outside the file is `xref.section-missing`'s, or the rebuild's when `startxref` names it | -| `stream.length-invalid` | A stream's `/Length` is not where its data ends: its `endstream` lies elsewhere and ends the data; or the `/Length` is no length — absent, not a non-negative integer, or naming an object the file lacks or that could not be read — and the `endstream` ends the data; or no `endstream` lies before the next object or the end of the file, no `endobj` of its object ends the data, and the data is the declared length, when it ends before that stop, or runs up to the next object; or the document's searches for `endstream` have read as much of the file as they may, and the data is taken without a search — but where a guard hid the `/Length`, whose data is then the guard's. A `/Length` that names an object a guard cut, or left out of the index, is not judged, though a missing `endstream` is reported all the same. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case. A cross-reference stream whose `/Length` the chain could not read, its data taken up to its first `endstream`, has it read once the chain is, unless a guard cut that data before every row the section declares — data that holds them all is checked whatever a guard did —: a `/Length` that is no length, or that no `endstream` confirms, is reported as any stream's, and one an `endstream` past the one the chain stopped at confirms is reported with the rows the chain left unread. Where the section's own row for that `/Length` lies among the rows the data holds, and gives a length an `endstream` confirms, the data is taken again to it before the rows are read, and nothing is reported; nor is anything of a `/Length` a guard cut or left out of the index | +| `stream.length-invalid` | A stream's `/Length` is not where its data ends: its `endstream` lies elsewhere and ends the data; or the `/Length` is no length — absent, not a non-negative integer, or naming an object the file lacks or that could not be read — and the `endstream` ends the data; or no `endstream` lies before the next object or the end of the file, no `endobj` of its object ends the data, and the data is the declared length, when it ends before that stop, or runs up to the next object; or the document's searches for `endstream` have read as much of the file as they may, and the data is taken without a search — but where a guard hid the `/Length`, whose data is then the guard's. A `/Length` that names an object a guard cut, or left out of the index, is not judged, though a missing `endstream` is reported all the same. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case. A cross-reference stream whose `/Length` the chain could not read, its data taken up to its first `endstream`, has it read once the chain is, unless a guard cut that data before every row the section declares — data that holds them all is checked whatever a guard did —: a `/Length` that is no length, or that no `endstream` confirms, is reported as any stream's, and one an `endstream` past the one the chain stopped at confirms is reported with the rows the chain left unread. Where the section's own row for that `/Length` lies among the rows the data holds — in a predicted row the spelled `endstream` parts too, the data decoded once more knowing its end hidden —, and gives a length an `endstream` confirms, the data is taken again to it before the rows are read, and nothing is reported; where that decoding reaches `MaxDecodedStreamLength` short of the row, the rows past the bound are the guard's, reported under `limit.decoded-stream`; nor is anything of a `/Length` a guard cut or left out of the index | | `stream.truncated` | A stream has no `endstream` before the next object or the end of the file, and its object's `endobj` ends its data — following the declared length, which is kept, or ending the object before that stop —; or neither, no declared length ends before the end of the file, and its data runs to it. Data taken without a search, the document's searches having read as much of the file as they may, is `stream.length-invalid`'s, to the end of the file too. [Streams whose length is wrong](#streams-whose-length-is-wrong), below, gives each case | | `stream.self-reference` | An object stream's dictionary names an object the stream holds — as its `/Type`, `/Length`, `/N`, `/First`, a filter or a parameter —: that object reads as null while the stream is decoded, and the stream is decoded without it | | `object-stream.member-moved` | An object is in the object stream its entry names, at another index than the entry gives, and was read where the stream's header lists it. Reported where the stream's data starts, the object, the stream and both indexes in the message | @@ -264,7 +264,7 @@ what it meets before is the file's and reported, its running out there — a los marker — is not, and where its output may lack what the run holds — Flate that lost its tail, LZW or RunLength data without its end marker — the output's end is cut, as above, for the predictor, the filters after it and an object stream's members. Flate data whose deflate body ends in the data as taken decodes whole, what it reports as read whole -but a checksum that disagrees only where it reaches into the run. The filters after a first whose output is whole +but a checksum the run cuts short, or that disagrees only where it reaches into the run. The filters after a first whose output is whole decode, and report, as read whole. ```text diff --git a/docs/website/docs/reference/reader-limits.md b/docs/website/docs/reference/reader-limits.md index 1150ac2a..97ccbd1b 100644 --- a/docs/website/docs/reference/reader-limits.md +++ b/docs/website/docs/reference/reader-limits.md @@ -145,12 +145,16 @@ there. Raise PdfReaderLimits.MaxDecodedStreamLength to read past it. section the guard cut, whatever type it gives, as read whole it is the section. What makes a section unreadable whatever a value a guard cut holds is the file's fault, as read whole: a value past the cut that no section read so far places, a `/W` that gives no three widths or none past 0, rows numbered wrongly — an `/Index` that holds no pairs, a - subsection whose elements are no integers, whose count is below 0, or that numbers past the object numbers, by both - its elements or by the one read whatever the cut one holds, a `/Size` with no `/Index` that gives no count —, as far as + subsection whose elements are no integers, whose first number or count is below 0, or that numbers past the object + numbers, by both its elements or by the one read whatever the cut one holds, a `/Size` with no `/Index` that gives no + count —, as far as the values read, written direct or through references, say. Data whose length the file gives, its filters written direct, decoded whole, that holds fewer rows than the counts read declare however few bytes the widths read leave a - row lacks them whatever the cut hides: the section is read as read whole, lacking rows, its trailer merged, the chain - going on and the index rebuilt. The rows a cross-reference stream's data lacks, + row lacks them whatever the cut hides: the section is read as read whole, lacking rows, its trailer merged and the + index rebuilt, and the object the guard cut is loaded once the chain is read, which reaches the guard. The rows the + data may hold at the fewest bytes a row may take, read with the cut value, are the guard's: the chain ends at the + section, as where rows a guard cut, and the validator says it unchecked; the chain goes on only past data that holds + none. The rows a cross-reference stream's data lacks, where its `/Length` so leads and its data was taken to the first `endstream`, are the guard's too, unless what follows that `endstream` proves it the stream's own, as above: they are then the file's to lack, and the index is rebuilt. An object the chain read through a reference the cut left whole is loaded once the chain is read only for a value it diff --git a/src/AdCodicem.Pdf/IO/PdfFileReader.cs b/src/AdCodicem.Pdf/IO/PdfFileReader.cs index 3367ff97..b2358db8 100644 --- a/src/AdCodicem.Pdf/IO/PdfFileReader.cs +++ b/src/AdCodicem.Pdf/IO/PdfFileReader.cs @@ -2913,6 +2913,7 @@ private bool TryReadXRefChain(long startOffset) } // A section a guard cut is where it was named, and is not looked for elsewhere: what lies near it would + // stand for it. if (section.CutByLimit || !TryRelocateXRefSection(offset, section, out previous, out hybrid)) { ReportLostSection(section, naming, offset, namedFrom); From 5e3133569292f9cd35f3e2576890d6548988891f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 08:09:36 +0000 Subject: [PATCH 141/146] test: hold where a deferred length's row is looked for again, and how far the guard lets rows read The last round of mutants found two rules of the deferred length's second decoding that no test held. The data is decoded again only where a subsection numbers the object the /Length names: one no row numbers is never looked for there, and a decoding guard that search would reach is not reported. Where that decoding stops at MaxDecodedStreamLength short of the /Length's row, the rows it holds are read as far as the guard lets them, a page's row in the parted predicted row among them, not only those the decoding before held. The RunLength chain's files share their predicted rows' writing. Refs #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 129 ++++++++++++++++-- .../DocumentReaderTests.cs | 50 +++++++ 2 files changed, 169 insertions(+), 10 deletions(-) diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 10852922..40185ab1 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -1002,36 +1002,145 @@ byte[] Data(long five, long stream) runs.AddRange("\nendstream\0\0\0\0"u8.ToArray()); runs.Add(rowBefore ? (byte)6 : (byte)13); runs.AddRange(Encode(rowBefore ? [(1, stream, 0)] : [(1, five, 0), (1, stream, 0)], Rows.Raw)); + return PredictedRunLength(runs, Columns); + } + + var length = Data(0, 0).Length.ToString(CultureInfo.InvariantCulture); + file.Object(5, direct ? "0" : length); + var at = file.Position; + file.Stream( + 7, + string.Create( + CultureInfo.InvariantCulture, + $"/Type /XRef /Size 266 /W [1 4 2] /Index {(rowBefore ? "[0 4 8 256 5 1 264 2 7 1]" : "[0 4 8 256 264 2 5 1 7 1]")} /Filter [/FlateDecode /RunLengthDecode] /DecodeParms [<< /Predictor 12 /Columns {Columns} >> null] /Root 1 0 R /Length {(direct ? length : "5 0 R")}"), + Data(file.OffsetOf(5), at)); + file.StartXRef(at); + return file.ToArray(); + } + + /// + /// As , the rows past the endstream they spell, but stream 7 numbers no + /// row for object 5, which no section places (/Index [0 4 8 256 264 2 7 1]): its /Length can be read once + /// the chain is, and the rows past the endstream are the file's to lack, the data up to it decoding to 1,052 bytes. + /// + public static byte[] DeferredLengthNoRowNumbersPastRunLengthRuns() + { + const int Columns = 45; + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + + // The data's length does not depend on the offsets its rows hold. + byte[] Data(long stream) + { + var runs = new List { 27 }; + runs.AddRange(Encode([(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, file.OffsetOf(3), 0)], Rows.Raw)); - while (runs.Count % Columns != 0) + for (var i = 0; i < 14; i++) { - runs.Add(128); + runs.AddRange([0x81, 0x00]); } - var predicted = new List(); + runs.Add(13); + runs.AddRange("\nendstream\0\0\0\0"u8.ToArray()); + runs.Add(6); + runs.AddRange(Encode([(1, stream, 0)], Rows.Raw)); + return PredictedRunLength(runs, Columns); + } - for (var i = 0; i < runs.Count; i += Columns) + file.Object(5, Data(0).Length.ToString(CultureInfo.InvariantCulture)); + var at = file.Position; + file.Stream( + 7, + string.Create( + CultureInfo.InvariantCulture, + $"/Type /XRef /Size 266 /W [1 4 2] /Index [0 4 8 256 264 2 7 1] /Filter [/FlateDecode /RunLengthDecode] /DecodeParms [<< /Predictor 12 /Columns {Columns} >> null] /Root 1 0 R /Length 5 0 R"), + Data(at)); + file.StartXRef(at); + return file.ToArray(); + } + + /// + /// A file of one revision: the catalog, the page tree, object 5, the page — object 155 — and cross-reference stream 7, + /// whose /Length names object 5 — or, with , gives the length —, and whose data a + /// RunLength step decodes after a Flate one, in a stored block with a PNG predictor of 45 columns. The first predicted + /// row decodes to the rows of objects 0 to 3 — 3 free — and 1,024 zeros; the second, which the endstream its rows spell + /// parts, to 5 zeros, the page's row, the 152nd, 1,057 bytes in, 630 zeros, then object 5's row, two of reserved types + /// that spell endstream after an end-of-line, and the stream's own row + /// (/Index [0 4 8 238 5 1 246 2 7 1]). The file is valid. + /// + public static byte[] DeferredLengthPageRowInPartedRunLengthRow(bool direct = false) + { + const int Columns = 45; + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [155 0 R] /Count 1 >>"); + + // The data's length does not depend on the offsets its rows hold. + byte[] Data(long five, long page, long stream) + { + var runs = new List { 27 }; + runs.AddRange(Encode([(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (0, 0, 0)], Rows.Raw)); + + for (var i = 0; i < 8; i++) { - predicted.Add(0); - predicted.AddRange(runs.GetRange(i, Columns)); + runs.AddRange([0x81, 0x00]); } - return Compress([.. predicted]); + runs.AddRange([0xFC, 0x00, 6]); + runs.AddRange(Encode([(1, page, 0)], Rows.Raw)); + + for (var i = 0; i < 5; i++) + { + runs.AddRange([0x83, 0x00]); + } + + runs.Add(6); + runs.AddRange(Encode([(1, five, 0)], Rows.Raw)); + runs.Add(13); + runs.AddRange("\nendstream\0\0\0\0"u8.ToArray()); + runs.Add(6); + runs.AddRange(Encode([(1, stream, 0)], Rows.Raw)); + return PredictedRunLength(runs, Columns); } - var length = Data(0, 0).Length.ToString(CultureInfo.InvariantCulture); + var length = Data(0, 0, 0).Length.ToString(CultureInfo.InvariantCulture); file.Object(5, direct ? "0" : length); + file.Object(155, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); var at = file.Position; file.Stream( 7, string.Create( CultureInfo.InvariantCulture, - $"/Type /XRef /Size 266 /W [1 4 2] /Index {(rowBefore ? "[0 4 8 256 5 1 264 2 7 1]" : "[0 4 8 256 264 2 5 1 7 1]")} /Filter [/FlateDecode /RunLengthDecode] /DecodeParms [<< /Predictor 12 /Columns {Columns} >> null] /Root 1 0 R /Length {(direct ? length : "5 0 R")}"), - Data(file.OffsetOf(5), at)); + $"/Type /XRef /Size 248 /W [1 4 2] /Index [0 4 8 238 5 1 246 2 7 1] /Filter [/FlateDecode /RunLengthDecode] /DecodeParms [<< /Predictor 12 /Columns {Columns} >> null] /Root 1 0 R /Length {(direct ? length : "5 0 R")}"), + Data(file.OffsetOf(5), file.OffsetOf(155), at)); file.StartXRef(at); return file.ToArray(); } + /// + /// Pads RunLength data with no-op bytes to whole rows of bytes, gives each row the PNG + /// predictor's None tag, and stores the rows in a zlib stream. + /// + private static byte[] PredictedRunLength(List runs, int columns) + { + while (runs.Count % columns != 0) + { + runs.Add(128); + } + + var predicted = new List(); + + for (var i = 0; i < runs.Count; i += columns) + { + predicted.Add(0); + predicted.AddRange(runs.GetRange(i, columns)); + } + + return Compress([.. predicted]); + } + /// /// A file of one cross-reference stream, object 7, whose /Length names object 5, which only the stream's own row /// places, before the rows of objects 1 and 2; two rows of reserved types spell endstream after an end-of-line diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 4294d3b1..79a12c02 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -3207,4 +3207,54 @@ public void A_decoding_guard_a_deferred_length_s_data_reaches_once_its_predicted var throwing = () => PdfDocument.Open(file, options with { ThrowOnLimit = true }); throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxDecodedStreamLength)); } + + [Fact] + public void Rows_a_decoding_guard_stops_once_a_parted_predicted_row_is_undone_are_read_as_far_as_the_guard_lets_them() + { + // As above, but the page, object 155, has its row in the predicted row the spelled endstream parts, the 152nd, 1,057 + // bytes into the rows, within the 1,100 bytes MaxDecodedStreamLength lets the RunLength step decode, and object 5's + // row past them. Read whole, the rows past the bound are the guard's, the page's row read. The data decoded again, + // its predicted row undone as far as it goes, stops at the bound short of the /Length's row: where the rows end is + // the guard's, the rows it holds read — not only those the decoding before, which dropped the parted row, held. + var options = new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxDecodedStreamLength = 1100 } }; + + using (var whole = PdfDocument.Open(ChainFiles.DeferredLengthPageRowInPartedRunLengthRow(direct: true), options)) + { + whole.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitDecodedStream); + whole.GetObject(new PdfObjectId(155)).AsDictionary().Required().IsOfType(PdfName.Page).Should().BeTrue(); + } + + var file = ChainFiles.DeferredLengthPageRowInPartedRunLengthRow(); + + using (var unbounded = PdfDocument.Open(file)) + { + unbounded.Diagnostics.Should().BeEmpty(); + } + + using var document = PdfDocument.Open(file, options); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitDecodedStream); + document.WasRepaired.Should().BeFalse(); + document.GetObject(new PdfObjectId(155)).AsDictionary().Required().IsOfType(PdfName.Page).Should().BeTrue(); + } + + [Fact] + public void A_deferred_length_no_row_numbers_is_not_looked_for_in_data_decoded_again() + { + // Stream 7's /Length names object 5, which no section places and no subsection of its own numbers, and its rows spell + // an endstream that parts a predicted row. The data up to it, its end the file's, decodes within the 1,100 bytes + // MaxDecodedStreamLength lets the RunLength step decode; decoded again, its predicted row undone, it would pass them, + // but no row could give the length there: what is read is what an unbounded reading reads, no guard reached. + var file = ChainFiles.DeferredLengthNoRowNumbersPastRunLengthRuns(); + string[] unboundedCodes; + + using (var unbounded = PdfDocument.Open(file)) + { + unboundedCodes = [.. unbounded.Diagnostics.Select(entry => entry.Code)]; + } + + using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxDecodedStreamLength = 1100 } }); + + document.Diagnostics.Select(entry => entry.Code).Should().Equal(unboundedCodes).And.NotContain(PdfDiagnosticCodes.LimitDecodedStream); + } } From d4683528e7bb46d01cf57bdfc702fe546708cdf4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 08:46:03 +0000 Subject: [PATCH 142/146] test: store the Flate data a MaxStreamLength cut is measured against Remote corpus run 78 failed one test, the deferred length checked on data MaxStreamLength cuts past every row, at a bound of 29 bytes. Its bounds were those of the Flate data this container's zlib writes at CompressionLevel.Optimal; the runtime Microsoft ships, as CI installs it, bundles zlib-ng, which writes other bytes, so 28 of them held fewer rows. The same test binary, run under that runtime here, fails as CI did, and passes under the distribution's. The file now holds its Flate data in a stored block for that test, the bounds 49, 50 and 60 counted from its layout, which no compressor moves. The test that bounds the decoded data keeps the compressed form, which does not depend on it. The whole unit suite passes under both runtimes. Refs #170, #220 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 14 ++++++++------ .../AdCodicem.Pdf.Tests/DocumentReaderTests.cs | 17 +++++++++-------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 40185ab1..3adb439d 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -641,12 +641,14 @@ byte[] Data(long four, long stream) /// /// A file of two revisions, as writes it, but stream 5 is - /// /Filter [/RunLengthDecode /FlateDecode]: a first literal run of RunLength holds the Flate data, which decodes to - /// its rows and 1,000 zero bytes, and a second the bytes of an end-of-line, endstream, another end-of-line and - /// XYZ, where a deferred search stops. Its /Length names object 4, which only the stream's own row places, - /// unless is false: written direct. The file is valid. + /// /Filter [/RunLengthDecode /FlateDecode]: literal runs of RunLength, of 128 bytes but the last, hold the Flate + /// data, which decodes to its rows and 1,000 zero bytes, and a last one the bytes of an end-of-line, endstream, + /// another end-of-line and XYZ, where a deferred search stops. With , the Flate data + /// holds them in a stored block — 7 bytes of header, then the rows —, so that what a cut of it leaves does not depend on + /// how a platform's zlib compresses. Its /Length names object 4, which only the stream's own row places, unless + /// is false: written direct. The file is valid. /// - public static byte[] UpdateOverRunsSpellingAnEndStream(bool deferred) + public static byte[] UpdateOverRunsSpellingAnEndStream(bool deferred, bool stored = false) { using var file = new Writer("1.5"); file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); @@ -661,7 +663,7 @@ byte[] Data(long four, long stream) Rows.Raw); using var output = new MemoryStream(); - using (var zlib = new ZLibStream(output, CompressionLevel.Optimal, leaveOpen: true)) + using (var zlib = new ZLibStream(output, stored ? CompressionLevel.NoCompression : CompressionLevel.Optimal, leaveOpen: true)) { zlib.Write([.. rows, .. new byte[1000]]); } diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 79a12c02..1cf12c37 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -2100,16 +2100,17 @@ public void A_deferred_length_s_data_that_holds_every_row_under_a_guard_on_decod } [Theory] - [InlineData(39, true)] - [InlineData(29, true)] - [InlineData(28, false)] + [InlineData(60, true)] + [InlineData(50, true)] + [InlineData(49, false)] public void A_deferred_length_s_data_MaxStreamLength_cuts_past_every_row_is_checked_against_its_length(int bound, bool everyRow) { - // As above, MaxStreamLength cutting the data past every row, from 29 bytes on: the run the cut parts is the guard's, - // but the length, checked against where the search found the endstream, which no cut moves, says the chain took the - // data short, as read whole. It was left unchecked, as data a guard cut that lacks rows is, which the cut keeps from - // being taken again (#220). - var file = ChainFiles.UpdateOverRunsSpellingAnEndStream(deferred: true); + // As above, MaxStreamLength cutting the data past every row, from 50 bytes on — a RunLength byte, 7 of the stored + // block's header and 42 of rows —: the run the cut parts is the guard's, but the length, checked against where the + // search found the endstream, which no cut moves, says the chain took the data short, as read whole. It was left + // unchecked, as data a guard cut that lacks rows is, which the cut keeps from being taken again (#220). The Flate data + // is stored: compressed, where a cut parts it depended on the platform's zlib, which zlib-ng writes otherwise. + var file = ChainFiles.UpdateOverRunsSpellingAnEndStream(deferred: true, stored: true); using var whole = PdfDocument.Open(file); using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = PdfReaderLimits.Default with { MaxStreamLength = bound } }); var invalid = whole.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.StreamLengthInvalid).Which; From 2055eebb183aeb65d9ff6b7a056734e4361d2af4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 09:54:17 +0000 Subject: [PATCH 143/146] test: hold the lines the batch's patch left partial, through documents or direct calls A coverage run of the unit suite on the committed corpus left 39 of the lines the batch changed partial or missed. Each one now has a test. A document takes the line where one reaches it; where no file can, the test calls the internal member directly. The new tests cover: - a rollback to a mark that records no decoding reports; - a range a source in memory lends nothing; - a window's edge outside the file; - a place outside the file said to end data beside white space; - data the file lost under the document; - a deferred length's chain of references one longer than a parse resolves; - sections that spend the endstream searches; - a section beside a cut /W whose data may hold rows; - ASCII85 white space under a hidden end. What no input takes stays partial: a null check on a set that is never null there, an operand a caller's precondition makes true, conditions that contradict each other, and the arms the compiler adds. Two faults the tests met are filed rather than pinned: a /Prev naming an object whose header a guard cut rebuilds the index once the chain is read, and a deferred length is reported past the chain's endstream once the searches are spent. Refs #170, #174, #220, #224, #263, #292, #293 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- tests/AdCodicem.Pdf.Tests/ChainFiles.cs | 149 +++++++ tests/AdCodicem.Pdf.Tests/DiagnosticsTests.cs | 70 ++++ .../DocumentReaderTests.cs | 160 ++++++++ .../AdCodicem.Pdf.Tests/GuardCutValueTests.cs | 385 +++++++++++++++++- tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs | 53 +++ 5 files changed, 816 insertions(+), 1 deletion(-) diff --git a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs index 3adb439d..1ac7a6c8 100644 --- a/tests/AdCodicem.Pdf.Tests/ChainFiles.cs +++ b/tests/AdCodicem.Pdf.Tests/ChainFiles.cs @@ -743,6 +743,60 @@ byte[] Build(long eightAt, out long eight) } } + /// + /// A file as writes it, object 4 cut before its value and the page's + /// row lacking, but for objects 8, 9 and 10, which follow the stream in that order and which only its rows place, in the + /// order its /Index gives — 9, 8, 10 —: the nearest has the second of those rows. Without + /// , the rows have none — /W [0 4 2], every row of type 1 — and object 0 has no row. + /// + public static byte[] AloneWithACutLengthBeforeObjectsItsRowsPlace(bool typeField) + { + // Every row and offset is as wide whatever it holds: where objects 8 to 10 lie is read once, then written. + _ = Build([0, 0, 0], out var after); + return Build(after, out _); + + byte[] Build(long[] placed, out long[] written) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + var four = file.Position; + var at = four + ("4 0 obj\n" + new string(' ', 1100) + "00\nendobj\n").Length; + (int Type, long Second, int Third)[] rows = + [ + (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0), (1, four, 0), (1, at, 0), (1, placed[1], 0), (1, placed[0], 0), + (1, placed[2], 0), (1, file.OffsetOf(3), 0), + ]; + var data = (typeField ? Encode([(0, 0, 65535), .. rows], Rows.Raw) : Untyped(rows))[..^3]; + file.Object(4, new string(' ', 1100) + data.Length.ToString("D2", CultureInfo.InvariantCulture)); + file.Stream( + 5, + $"/Type /XRef /Size 11 /W [{(typeField ? "1" : "0")} 4 2] /Index [{(typeField ? "0 3" : "1 2")} 4 2 9 1 8 1 10 1 3 1] /Root 1 0 R /Length 4 0 R", + data); + written = [file.Position, 0, 0]; + file.Object(8, "(eight)"); + written[1] = file.Position; + file.Object(9, "(nine)"); + written[2] = file.Position; + file.Object(10, "(ten)"); + file.StartXRef(at); + return file.ToArray(); + } + + static byte[] Untyped((int Type, long Second, int Third)[] rows) + { + var bytes = new List(); + + foreach (var (_, second, third) in rows) + { + bytes.AddRange([(byte)(second >> 24), (byte)(second >> 16), (byte)(second >> 8), (byte)second, (byte)(third >> 8), (byte)third]); + } + + return [.. bytes]; + } + } + /// /// A file of one cross-reference stream, object 7, in ASCII85, whose /Length names object 5, which only the /// stream's own row places, holding the length after 1,015 spaces: a MaxObjectLength of 1,024 parts it after its @@ -1647,6 +1701,101 @@ public static byte[] UpdatesWithDeferredLengthsBeforeWhiteSpace(int count, int r static string Invariant(FormattableString text) => text.ToString(CultureInfo.InvariantCulture); } + /// + /// A file of revisions, newest first in the file, each a cross-reference stream whose + /// /Length names an object only its own rows place, and each naming the next through /Prev, 1,024 spaces + /// past the rows of the one before — more than a search for a misplaced section spans —: the data of each runs over + /// the streams after it and spaces to the one endstream they share, at the end of the + /// file, as its length gives. Searched for it one after the other, past their first window, the streams read the file + /// about once each. With , the oldest revision comes first in the file, before them: stream + /// 5, whose /Length, object 4, which only its own rows place, holds the length after 1,100 spaces — a + /// MaxObjectLength of 1,024 cuts it before its value —, whose rows of objects 4, 5 and 3 lose their last 3 bytes, + /// the page's row lacking, and whose endstream 100 spaces part from its endobj, which object 20 follows — + /// or, with , 60 regular characters glued to it, which make it no keyword, then an end-of-line. + /// The newest stream indexes the catalog and the page tree, and the page where no such revision does. + /// + public static byte[] UpdatesSpendingTheSearches(int spenders, int padding, bool cutTarget, bool glued = false) + { + // Every offset and length is written ten digits wide: where each stream and its data lie is read once, then written. + var previous = new long[spenders]; + var lengths = new long[spenders]; + _ = Build(out var at, out var data, out var target, out var end); + + for (var i = 0; i < spenders; i++) + { + previous[i] = i + 1 < spenders ? at[i + 1] : target; + lengths[i] = end - data[i]; + } + + return Build(out _, out _, out _, out _); + + byte[] Build(out long[] streams, out long[] starts, out long oldest, out long shared) + { + using var file = new Writer("1.5"); + file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"); + file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>"); + oldest = -1; + + if (cutTarget) + { + file.Object(4, new string(' ', 1100) + "18"); + oldest = file.Position; + file.Text("5 0 obj\n<< /Type /XRef /Size 6 /W [1 4 2] /Index [4 2 3 1] /Root 1 0 R /Length 4 0 R >>\nstream\n"); + file.Bytes(Encode([(1, file.OffsetOf(4), 0), (1, oldest, 0), (1, file.OffsetOf(3), 0)], Rows.Raw)[..^3]); + file.Text("\nendstream" + (glued ? new string('x', 60) + "\n" : new string(' ', 100)) + "endobj\n"); + } + + for (var i = 0; i < spenders; i++) + { + file.Object(20 + (2 * i), lengths[i].ToString("D10", CultureInfo.InvariantCulture)); + } + + streams = new long[spenders]; + starts = new long[spenders]; + + for (var i = 0; i < spenders; i++) + { + // The data of the stream before takes in 1,024 spaces, more than a search for a misplaced section spans. + if (i > 0) + { + file.Text(new string(' ', 1024)); + } + + var length = 20 + (2 * i); + streams[i] = file.Position; + var rows = new List<(int Type, long Second, int Third)>(); + + if (i == 0) + { + rows.AddRange([(0, 0, 65535), (1, file.OffsetOf(1), 0), (1, file.OffsetOf(2), 0)]); + + if (!cutTarget) + { + rows.Add((1, file.OffsetOf(3), 0)); + } + } + + var index = string.Create(CultureInfo.InvariantCulture, $"{(i == 0 ? $"0 {rows.Count} " : string.Empty)}{length} 2"); + var prev = i + 1 < spenders || cutTarget + ? string.Create(CultureInfo.InvariantCulture, $" /Prev {previous[i]:D10}") + : string.Empty; + rows.AddRange([(1, file.OffsetOf(length), 0), (1, streams[i], 0)]); + file.Text(string.Create( + CultureInfo.InvariantCulture, + $"{length + 1} 0 obj\n<< /Type /XRef /Size 40 /W [1 4 2] /Index [{index}] /Root 1 0 R{prev} /Length {length} 0 R >>\nstream\n")); + starts[i] = file.Position; + file.Bytes(Encode([.. rows], Rows.Raw)); + } + + file.Text(new string(' ', padding)); + shared = file.Position; + file.Text("\nendstream\nendobj\n"); + file.StartXRef(spenders > 0 ? streams[0] : oldest); + return file.ToArray(); + } + } + private enum Rows { Raw, diff --git a/tests/AdCodicem.Pdf.Tests/DiagnosticsTests.cs b/tests/AdCodicem.Pdf.Tests/DiagnosticsTests.cs index ce886667..f90b357f 100644 --- a/tests/AdCodicem.Pdf.Tests/DiagnosticsTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DiagnosticsTests.cs @@ -149,6 +149,76 @@ public void A_copy_of_a_report_the_capacity_left_out_is_counted_again_each_time_ target.SuppressedCount.Should().Be(3); } + [Fact] + public void Rolling_back_beside_the_guards_keeps_their_reports_and_counts_those_the_capacity_left_out() + { + // A reading met two guards beside faults it is to make again, past a capacity of three: the guards stand, the one + // the capacity left no room for counted as left out still, and the faults are dropped, the one the capacity left + // out no longer counted. A mark that records no decoding reports, which the reader never rolls back to beside its + // guards, leaves which reports decodings moved in as it stands: a fault a decoding moved in before the mark is not + // moved in again. + var target = new PdfDiagnostics { Capacity = 3 }; + var decodings = target.Decodings; + var before = decodings.GetMark(); + decodings.Warn(PdfDiagnosticCodes.FilterFailed, "Moved in before the mark.", 3); + decodings.MoveDecodingReportsTo(target, before); + + var mark = target.GetMark(); + target.Warn("dropped", "Met beside the guards."); + target.ReachLimit(PdfDiagnosticCodes.LimitDecodedStream, "Kept.", 7); + target.Warn("left.out", "Past the capacity."); + target.ReachLimit(PdfDiagnosticCodes.LimitObject, "Past the capacity.", 9); + target.RollBackBesideLimits(mark); + + target.Select(d => d.Code).Should().Equal(PdfDiagnosticCodes.FilterFailed, PdfDiagnosticCodes.LimitDecodedStream); + target.LimitsReached.Should().Be(2); + target.SuppressedCount.Should().Be(1); + + var again = decodings.GetMark(); + decodings.Warn(PdfDiagnosticCodes.FilterFailed, "Moved in before the mark.", 3); + decodings.MoveDecodingReportsTo(target, again); + + target.Select(d => d.Code).Should().Equal(PdfDiagnosticCodes.FilterFailed, PdfDiagnosticCodes.LimitDecodedStream); + target.SuppressedCount.Should().Be(1); + } + + [Fact] + public void Rolling_back_beside_the_guards_holds_again_as_moved_in_only_the_guard_reports_a_decoding_moved_in() + { + // A guard reported outside any decoding before any decoding has reported — which the reader never rolls back beside + // to such a mark, a decoding having reported by then —, then a guard a decoding reached beside a fault, are each + // rolled back beside to a mark that records which reports decodings moved in. Every guard's report stands. The + // decoding's is held again as moved in, so that a decoding that meets it again does not report it twice; the fault + // dropped with the rollback is forgotten, so that a decoding that meets it again moves it in again (#144, #220). + var target = new PdfDiagnostics(); + var decodings = target.Decodings; + + var reading = target.GetMarkWithDecodingReports(); + target.ReachLimit(PdfDiagnosticCodes.LimitObject, "The reader's guard.", 9); + target.Warn("dropped", "Met beside the reader's guard."); + target.RollBackBesideLimits(reading); + + target.Select(d => d.Message).Should().Equal("The reader's guard."); + + var decoded = target.GetMarkWithDecodingReports(); + var first = decodings.GetMark(); + decodings.ReachLimit(PdfDiagnosticCodes.LimitDecodedStream, "The decoding's guard.", 7); + decodings.Warn(PdfDiagnosticCodes.FilterFailed, "Met beside the decoding's guard.", 7); + decodings.MoveDecodingReportsTo(target, first); + target.RollBackBesideLimits(decoded); + + target.Select(d => d.Message).Should().Equal("The reader's guard.", "The decoding's guard."); + + var again = decodings.GetMark(); + decodings.ReachLimit(PdfDiagnosticCodes.LimitDecodedStream, "The decoding's guard.", 7); + decodings.Warn(PdfDiagnosticCodes.FilterFailed, "Met beside the decoding's guard.", 7); + decodings.MoveDecodingReportsTo(target, again); + + target.Select(d => d.Message).Should().Equal("The reader's guard.", "The decoding's guard.", "Met beside the decoding's guard."); + target.LimitsReached.Should().Be(3); + target.SuppressedCount.Should().Be(0); + } + [Fact] public void A_nested_mark_keeps_or_drops_only_what_came_after_it() { diff --git a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs index 1cf12c37..35dbd677 100644 --- a/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs +++ b/tests/AdCodicem.Pdf.Tests/DocumentReaderTests.cs @@ -678,6 +678,26 @@ public void A_source_gives_an_empty_window_for_a_range_outside_itself_or_of_no_l window.Offset.Should().Be(0); } + [Theory] + [InlineData(-1, 2, false)] + [InlineData(0, -1, false)] + [InlineData(3, 2, false)] + [InlineData(5, 0, false)] + [InlineData(1, 3, true)] + [InlineData(4, 0, true)] + public void A_source_in_memory_lends_only_a_range_it_holds_whole(long offset, int length, bool lent) + { + // A stream's data is lent where it lies in a file in memory (#170). The reader asks only for data that starts in the + // file and whose length it clamped to the file, which no file can make otherwise: a range before the source, of a + // negative length, or running past its end is lent nothing, rather than sliced out of bytes the source does not hold. + // An empty range at its end is lent as empty. + var data = "%PDF"u8.ToArray(); + using var source = PdfFileSource.FromMemory(data); + + source.TryLend(offset, length, out var memory).Should().Be(lent); + memory.ToArray().Should().Equal(lent ? data.AsSpan((int)offset, length).ToArray() : []); + } + [Fact] public void A_stream_s_data_is_read_from_the_file_each_time_it_is_asked_for_and_kept_by_nothing() { @@ -1881,6 +1901,37 @@ public void A_deferred_length_is_taken_again_by_a_row_with_no_type_field() document.Diagnostics.Should().NotContain(entry => entry.Code == PdfDiagnosticCodes.StreamLengthInvalid); } + [Theory] + [InlineData(32)] + [InlineData(33)] + public void A_deferred_length_is_taken_again_through_as_many_references_as_the_parse_resolves_a_length_through_and_no_more(int objects) + { + // Stream 7's /Length names object 4, which only its own row places, before rows that spell an endstream; object 4 + // names object 100, which names the next, and so on, the newer table placing them, to the last, which holds the + // length. Through 32 objects, as far as the parse resolves a /Length, the data is taken again to that length, and + // every row is read. Through one more, the chain reaches no length, as a whole read of the stream reaches none: the + // data is left as the chain took it, up to the spelled endstream, and the rows past it are the file's to lack — the + // section incomplete, the length reported once the chain is read, the index rebuilt to find the page (#220). + var file = ChainFiles.UpdatePlacingAPaddedLengthChain(objects, ownRow: true); + using var document = PdfDocument.Open(file); + var section = document.Reader.Structure.Sections[1]; + + document.GetObject(new PdfObjectId(3)).AsDictionary().Required().IsOfType(PdfName.Page).Should().BeTrue(); + + if (objects == 32) + { + section.Incomplete.Should().BeFalse(); + document.Diagnostics.Should().BeEmpty(); + return; + } + + section.Incomplete.Should().BeTrue(); + section.Fault.Should().Be("it holds 5 rows where its /Index and /Size declare 8"); + document.WasRepaired.Should().BeTrue(); + document.Diagnostics.Should().ContainSingle(entry => entry.Code == PdfDiagnosticCodes.StreamLengthInvalid).Which.Message.Should().Be( + "The /Length of cross-reference stream 7 names object 4 0, which holds null, not a length; the chain took its data up to its endstream, after 35 bytes."); + } + [Fact] public void A_deferred_length_a_newer_section_refused_takes_no_data_again() { @@ -2252,6 +2303,31 @@ public void Updates_each_with_a_long_deferred_length_are_all_searched() source.BytesRead.Should().BeLessThan(3L * file.Length); } + [Theory] + [InlineData(1)] + [InlineData(5)] + public void Reports_the_data_a_deferred_length_gives_past_the_chain_s_endstream_once_the_searches_have_read_all_they_may(int sections) + { + // Cross-reference streams written one after the other, with no endstream of their own, share 512 KB of data and the + // endstream after it, which four zero bytes and a second endstream follow; each one's /Length names an object only + // the oldest section, a classic table, places. Each is searched past its window up to the first endstream, and five + // such searches read more than the four times the file the document's searches may (EndStreamSearchPasses). The + // newest one's length, read once the chain is, gives the data up to the second endstream: what the chain did not read + // is reported, as for one such stream alone, though the file may no longer be read to find where the first starts. + var file = SectionsSharingTheirData(sections, 512 * 1024, newestExtra: 14); + using var document = PdfDocument.Open(file); + + document.WasRepaired.Should().BeFalse(); + document.Reader.Structure.Sections.Should().HaveCount(sections + 1).And.OnlyContain(section => section.State == XRefSectionState.Read); + (document.Reader.EndStreamSearchBytes >= PdfFileReader.EndStreamSearchPasses * file.Length).Should().Be(sections == 5); + var report = document.Diagnostics.Should().ContainSingle().Which; + report.Code.Should().Be(PdfDiagnosticCodes.StreamLengthInvalid); + report.Position.Should().Be(PdfTemplate.OffsetOf(file, ">>\nstream\n", sections) + ">>\nstream\n".Length); + report.Message.Should().Be(string.Create( + CultureInfo.InvariantCulture, + $"The /Length of cross-reference stream {40 + sections} gives 524305 bytes, which an endstream confirms; the chain, which could not read it, took the data up to an endstream 524291 bytes in, and read no row past it.")); + } + [Theory] [InlineData(false, false, "\n")] [InlineData(true, false, "\n")] @@ -2342,6 +2418,33 @@ public void A_deferred_length_s_data_with_no_endstream_before_the_next_object_ma $"its /Length is the reference 9 0 R, which no section read before it places where it can be read, and no endstream follows its data before the next object, at {next}")); } + [Fact] + public void A_deferred_length_s_data_the_searches_could_no_longer_read_makes_its_section_malformed_and_says_why() + { + // Six revisions, newest first in the file, each a cross-reference stream whose /Length only its own rows place, and + // whose data runs over the streams after it and 500,000 spaces to the endstream they share at the end of the file: + // the first five are each searched for it past their first window, and read the file about once each, more than + // four times in all — as much as the document's searches may (EndStreamSearchPasses). The sixth's data is searched + // in the bytes read with its object alone, which hold no endstream: it ends nowhere the section says, and the fault + // says the search was not made, not that no endstream follows the data (#220). + var file = ChainFiles.UpdatesSpendingTheSearches(spenders: 6, padding: 500_000, cutTarget: false); + using var document = PdfDocument.Open(file); + + var sections = document.Reader.Structure.Sections; + sections.Should().HaveCount(6); + sections.Take(5).Should().OnlyContain(section => section.State == XRefSectionState.Read && !section.Incomplete); + sections[5].State.Should().Be(XRefSectionState.Malformed); + sections[5].CutByLimit.Should().BeFalse(); + const string Fault = + "its /Length is the reference 30 0 R, which no section read before it places where it can be read, and its data was not searched for its endstream, the document's searches for endstream having read as much of the file as they may"; + sections[5].Fault.Should().Be(Fault); + document.Reader.EndStreamSearchBytes.Should().BeGreaterThanOrEqualTo(PdfFileReader.EndStreamSearchPasses * file.Length); + document.Diagnostics.Should().ContainSingle().Which.Should().Match(diagnostic => + diagnostic.Code == PdfDiagnosticCodes.XRefSectionUnreadable && diagnostic.Position == sections[5].Offset && + diagnostic.Message.Contains(Fault, StringComparison.Ordinal)); + document.GetObject(new PdfObjectId(3)).AsDictionary().Required().IsOfType(PdfName.Page).Should().BeTrue(); + } + [Fact] public void A_deferred_length_read_once_the_chain_is_says_what_its_chain_leads_to() { @@ -3015,6 +3118,63 @@ private static byte[] ChainOfSections(int sections, int rows) return Encoding.ASCII.GetBytes(file.ToString()); } + /// + /// A classic table placing the catalog, an empty page tree and objects 31 onward, then + /// cross-reference streams, 41 onward, each naming the one before it, the oldest the table, written one after the other + /// with no endstream of its own: the data of each, its one free row first, runs through the streams after it and + /// bytes of x to the endstream they share, which four zero bytes and a second + /// endstream follow. Each stream's /Length is the object the table places for it, 31 onward, which gives + /// its data up to the first endstream; the newest one's gives bytes more. + /// + private static byte[] SectionsSharingTheirData(int count, int padding, int newestExtra) + { + // Each length is written in ten digits: the offsets the first pass finds hold for the second, which writes them. + var lengths = new long[count]; + var bytes = Array.Empty(); + + for (var pass = 0; pass < 2; pass++) + { + var file = new HandWrittenFile(version: "1.5"); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + }; + + for (var i = 0; i < count; i++) + { + rows[31 + i] = file.Object(31 + i, lengths[i].ToString("D10", CultureInfo.InvariantCulture)); + } + + var previous = file.Table(31 + count, rows, string.Create(CultureInfo.InvariantCulture, $"/Size {31 + count} /Root 1 0 R")); + var dataStarts = new long[count]; + + for (var i = 0; i < count; i++) + { + var at = file.Position; + file.Raw(string.Create( + CultureInfo.InvariantCulture, + $"{41 + i} 0 obj\n<< /Type /XRef /Size 60 /W [1 1 1] /Index [{51 + i} 1] /Root 1 0 R /Prev {previous} /Length {31 + i} 0 R >>\nstream\n")); + dataStarts[i] = file.Position; + file.Raw("\0\0\0"); + previous = at; + } + + file.Raw(new string('x', padding)); + var dataEnd = file.Position; + file.Raw("\nendstream\0\0\0\0\nendstream\nendobj\n"); + file.Raw(string.Create(CultureInfo.InvariantCulture, $"startxref\n{previous}\n%%EOF\n")); + bytes = file.Bytes; + + for (var i = 0; i < count; i++) + { + lengths[i] = dataEnd - dataStarts[i] + (i == count - 1 ? newestExtra : 0); + } + } + + return bytes; + } + /// Where the data of the file's object stream starts. private static long ObjectStreamDataStart(byte[] file) { diff --git a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs index 29dea14b..c1bebbb6 100644 --- a/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs +++ b/tests/AdCodicem.Pdf.Tests/GuardCutValueTests.cs @@ -1493,6 +1493,40 @@ public void Data_that_is_all_the_run_a_hidden_length_may_leave_out_is_the_guard_ guarded.Diagnostics.Select(entry => entry.Code).Should().Equal(GuardCode(guard)); } + [Theory] + [InlineData("cut", "/ASCII85Decode", " \t\n")] + [InlineData("left out", "/ASCII85Decode", "\r\n\f\n")] + [InlineData("left out", "[/ASCII85Decode /FlateDecode]", " \n")] + [InlineData("cut", "/ASCIIHexDecode", " %c\n")] + public void Data_that_is_all_the_run_a_hidden_length_may_leave_out_decodes_whole_to_nothing_under_ascii85_or_asciihex( + string guard, string filter, string tail) + { + // Content stream 4's /Length is 0, then nothing but white space, or under ASCIIHex a comment past it, before the + // endstream the next object proves its own, under a /Length a guard cut or left out of the index: the search took + // the run the hidden length may leave out for the data (#224). White space decodes to nothing in ASCII85 or ASCIIHex + // data, and so does a comment in ASCIIHex, whatever the hidden length holds of them: what the stream decodes to, + // nothing, is whole, as read whole, for a Flate filter after ASCII85 too, and no end marker the run lacks is + // reported. Under a filter with an end, the run is the guard's to lack (above). + var file = HiddenLengthFile(guard, "/Filter " + filter, [], tail); + + using (var document = PdfDocument.Open(file)) + { + document.GetObject(new PdfObjectId(4)).AsStream().Required().Decode().Length.Should().Be(0); + document.Diagnostics.Should().BeEmpty(); + } + + using var guarded = PdfDocument.Open(file, HiddenLengthOptions(guard)); + var stream = guarded.GetObject(new PdfObjectId(4)).AsStream().Required(); + var decoded = PdfFilterPipeline.Decode(stream, null, out var leftEncoded, out var cutByGuard); + + stream.Data.EndHidden.Should().BeTrue(); + stream.GetRawBytes().ToArray().Should().Equal(Encoding.Latin1.GetBytes(tail[..^1])); + decoded.Length.Should().Be(0); + leftEncoded.Should().BeFalse(); + cutByGuard.Should().BeFalse(); + guarded.Diagnostics.Select(entry => entry.Code).Should().Equal(GuardCode(guard)); + } + /// /// A file whose content stream 4 holds and before its endstream, which /// object 5 after it proves its own, its /Length object 20, which a MaxObjectLength of 1,024 cuts, or, left @@ -1585,6 +1619,31 @@ public void An_endstream_where_a_guard_hid_the_length_ends_data_that_is_the_file } } + [Theory] + [InlineData("cut")] + [InlineData("left out")] + public void An_endstream_the_file_ends_with_ends_data_that_is_the_file_s_where_a_guard_hid_the_length(string guard) + { + // Object 4's /Length, object 20, is one MaxObjectLength cuts or MaxIndexEntries leaves out of the index, and the file + // ends on its endstream, with no end-of-line after it: the keyword, which no byte goes on, is whole, and the end of + // the file that follows proves it the stream's own. The data is the file's, and its object lacks its endobj, as where + // an end-of-line ends the file (#224). + var file = StreamFollowedBy(guard, "end of file", endObj: false)[..^1]; + var limits = guard == "cut" + ? PdfReaderLimits.Default with { MaxObjectLength = 1024 } + : PdfReaderLimits.Default with { MaxIndexEntries = 20 }; + using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = limits }); + + var stream = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + Encoding.Latin1.GetString(file).Should().EndWith("\nendstream"); + stream.GetRawBytes().ToArray().Should().Equal("BT ET"u8.ToArray()); + stream.Data.CutByGuard.Should().BeFalse(); + document.Reader.IsEndObjMissing(4, out _).Should().BeTrue(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal( + guard == "cut" ? PdfDiagnosticCodes.LimitObject : PdfDiagnosticCodes.LimitIndexEntries); + } + [Theory] [InlineData(false)] [InlineData(true)] @@ -2039,6 +2098,101 @@ public void The_proof_of_an_endstream_weighs_each_caller_s_next_object_against_w provider.IsOwnEndStream(data.Position + data.Length, EndObjState.Present, six + 1).Should().BeFalse(); } + [Fact] + public void The_proof_of_an_endstream_said_to_be_followed_by_an_endobj_proves_nothing_where_none_can_be_read() + { + // Object 4's endstream ends the file but for an end-of-line, or is followed by its endobj, which ends the file. Told + // that an endobj follows the keyword, the proof reads on past it to that endobj: none follows where the file ends, + // nor where its source, cut short after the document opened, gives no byte past that end-of-line, and nothing is + // proven — where the file holds one, the end of the file after it proves the endstream the stream's own (#224). + var bare = StreamFollowedBy("cut", "end of file", endObj: false); + var ended = StreamFollowedBy("cut", "end of file", endObj: true); + var dataEnd = (long)Encoding.Latin1.GetString(ended).LastIndexOf("\nendstream", StringComparison.Ordinal); + using var withoutEndObj = PdfDocument.Open(bare, ObjectGuard); + using var whole = PdfDocument.Open(ended, ObjectGuard); + using var source = new CutShortSource(ended); + using var cut = PdfDocument.Open(source, ObjectGuard, ownsSource: false); + source.CutAt((int)dataEnd + "\nendstream\n".Length); + + ((IPdfStreamDataProvider)withoutEndObj.Reader).IsOwnEndStream(dataEnd, EndObjState.Present, null).Should().BeFalse(); + ((IPdfStreamDataProvider)withoutEndObj.Reader).IsOwnEndStream(dataEnd, EndObjState.Absent, null).Should().BeTrue(); + ((IPdfStreamDataProvider)cut.Reader).IsOwnEndStream(dataEnd, EndObjState.Present, null).Should().BeFalse(); + ((IPdfStreamDataProvider)whole.Reader).IsOwnEndStream(dataEnd, EndObjState.Present, null).Should().BeTrue(); + } + + [Fact] + public void No_place_outside_the_file_is_said_to_end_data_beside_white_space() + { + // Content stream 4, whose /Length MaxObjectLength cut, ends its data in a space right before an endstream object 5 + // proves its own: only the hidden length says whether the space is the data's, so the end may lie beside it. No search + // ends data before the file's first byte or past its last, and there no byte says white space lies beside it (#224). + var file = HexDataEndingInASpaceBeforeAProvenEndStream(); + using var document = PdfDocument.Open(file, ObjectGuard); + var data = document.GetObject(new PdfObjectId(4)).AsStream().Required().Data; + var provider = (IPdfStreamDataProvider)document.Reader; + + data.EndHidden.Should().BeTrue(); + provider.EndsBesideWhiteSpace(data.Position + data.Length).Should().BeTrue(); + provider.EndsBesideWhiteSpace(-1).Should().BeFalse(); + provider.EndsBesideWhiteSpace(0).Should().BeFalse(); + provider.EndsBesideWhiteSpace(file.Length + 1L).Should().BeFalse(); + } + + [Fact] + public void Data_whose_last_byte_the_file_lost_under_the_document_is_no_longer_said_to_end_where_the_guard_hides() + { + // Content stream 4, whose /Length MaxObjectLength cut, ends its data in a space right before an endstream object 5 + // proves its own: its end is the guard's to hide. The file then loses that space under the document, and the stream, + // which the cache let go for 64 other objects, is parsed again: the search and its proof are not read again, but no + // byte says white space ends the data any longer, and its end is not hidden; its data is what the file still holds + // (#224). + var file = HexDataEndingInASpaceBeforeAProvenEndStream(others: 64); + var source = new CutShortSource(file); + using var document = PdfDocument.Open(source, ObjectGuard with { ObjectCacheCapacity = 64 }, ownsSource: true); + var first = document.GetObject(new PdfObjectId(4)).AsStream().Required().Data; + first.EndHidden.Should().BeTrue(); + + for (var number = 30; number < 30 + 64; number++) + { + _ = document.GetObject(new PdfObjectId(number)); + } + + source.CutAt((int)(first.Position + first.Length - 1)); + var again = document.GetObject(new PdfObjectId(4)).AsStream().Required(); + + again.Data.Position.Should().Be(first.Position); + again.Data.EndHidden.Should().BeFalse(); + Encoding.ASCII.GetString(again.GetRawBytes().Span).Should().Be("48656C6C6F"); + } + + /// + /// A page whose content stream, object 4, holds 48656C6C6F under /ASCIIHexDecode, the space right before + /// its endstream, which object 5, after it, proves its own; its /Length, object 20, holds 11 after 2,000 + /// spaces, which cuts; then strings, objects 30 and up. + /// + private static byte[] HexDataEndingInASpaceBeforeAProvenEndStream(int others = 0) + { + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, Page), + [20] = file.Object(20, new string(' ', 2000) + "11"), + [4] = file.Object(4, "<< /Length 20 0 R /Filter /ASCIIHexDecode >>\nstream\n48656C6C6F endstream"), + [5] = file.Object(5, "(five)"), + }; + + for (var number = 30; number < 30 + others; number++) + { + rows[number] = file.Object(number, Invariant($"({number})")); + } + + var size = 30 + others; + file.Table(size, rows, Invariant($"/Size {size} /Root 1 0 R")); + return file.Bytes; + } + [Theory] [InlineData("xref")] [InlineData("startxref")] @@ -2081,6 +2235,67 @@ public void Rows_a_cross_reference_stream_whose_length_a_guard_cut_lacks_are_the document.GetObject(new PdfObjectId(5)).AsStream().Required().Data.CutByGuard.Should().BeFalse(); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public void The_nearest_object_a_cross_reference_stream_s_rows_place_past_its_data_proves_its_endstream_whatever_their_order(bool typeField) + { + // As above, but for objects 8, 9 and 10, which follow stream 5 in that order and which only its rows place, 9 first, + // then 8, then 10 — with a type field, or without, every row then of type 1. The object that follows the stream's + // endobj is the nearest of them, 8, which proves the endstream its own whichever of the rows places it: the row the + // data lacks is the file's to lack, as read whole, the index rebuilt, the page read (#224). + var file = ChainFiles.AloneWithACutLengthBeforeObjectsItsRowsPlace(typeField); + using var whole = PdfDocument.Open(file); + using var document = PdfDocument.Open(file, ObjectGuard); + + whole.Reader.Structure.Sections[0].Incomplete.Should().BeTrue(); + document.Reader.Structure.Sections[0].Incomplete.Should().BeTrue(); + document.Reader.Structure.Sections[0].CutByLimit.Should().BeFalse(); + document.GetObject(new PdfObjectId(3)).AsDictionary().Required().IsOfType(PdfName.Page).Should().BeTrue(); + document.WasRepaired.Should().BeTrue(); + document.GetObject(new PdfObjectId(8)).Should().BeOfType().Which.ToText().Should().Be("eight"); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject, PdfDiagnosticCodes.XRefRebuilt); + } + + [Theory] + [InlineData(1, false)] + [InlineData(5, false)] + [InlineData(1, true)] + [InlineData(5, true)] + public void Rows_a_cross_reference_stream_whose_length_a_guard_cut_lacks_are_the_guard_s_where_nothing_read_proves_its_endstream(int updates, bool glued) + { + // Stream 5's /Length, object 4, which only its own row places, is one MaxObjectLength cuts before its value, and its + // data lacks the page's row; 100 spaces part its endstream from its endobj, which the update's first object follows. + // Read on to, that proves the endstream its own: the row is the file's to lack, as read whole, the index rebuilt, the + // page read. Five updates whose data runs over 500,000 spaces to an endstream they share read the file about once + // each, as much as the document's searches may (EndStreamSearchPasses): nothing is read past stream 5's endstream, + // which nothing then proves its own, and where its data ends is the guard's — the page's row with it, which reads as + // null until the guard is lifted. So it is where 60 characters glued to the endstream make it no keyword, which + // proves nothing, whether the searches can still read past it or not (#224). + var exhausted = updates == 5; + var guards = exhausted || glued; + var file = ChainFiles.UpdatesSpendingTheSearches(updates, padding: 500_000, cutTarget: true, glued); + using var document = PdfDocument.Open(file, ObjectGuard); + var section = document.Reader.Structure.Sections[^1]; + + (document.Reader.EndStreamSearchBytes >= PdfFileReader.EndStreamSearchPasses * file.Length).Should().Be(exhausted); + document.Reader.Structure.Sections.Should().HaveCount(updates + 1).And.OnlyContain(entry => entry.State == XRefSectionState.Read); + section.CutByLimit.Should().Be(guards); + section.Incomplete.Should().Be(!guards); + + if (guards) + { + document.GetObject(new PdfObjectId(3)).Should().BeOfType(); + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + return; + } + + document.GetObject(new PdfObjectId(3)).AsDictionary().Required().IsOfType(PdfName.Page).Should().BeTrue(); + document.WasRepaired.Should().BeTrue(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject, PdfDiagnosticCodes.XRefRebuilt); + } + [Theory] [InlineData(false)] [InlineData(true)] @@ -2724,6 +2939,54 @@ public void A_section_offset_a_guard_cut_ends_the_chain_as_the_guard_s(string ke throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); } + [Fact] + public void A_prev_naming_an_object_whose_header_a_guard_cut_ends_the_chain_as_the_guard_s() + { + // The newer table's /Prev is object 9, which holds the older table's offset; 3,000 blanks part its generation from + // its obj, white space a header may hold, and MaxObjectLength cuts the header itself before its keyword: the chain + // reads nothing of the object, its value no more than its header. The chain ends there for the guard, as where the + // guard cuts the value, and the guard is reached once the chain is read, where loading object 9 reaches it — thrown at + // opening when the caller asks for it (#263). The validation says the older sections, and the header of object 9, + // were not checked. That load then rebuilds the index, as a read of a header past the guard does, which this test + // leaves aside (#292). + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + [3] = file.Object(3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 1 1] /Resources << >> >>"), + }; + var older = file.Table(4, rows, "/Size 4 /Root 1 0 R"); + var four = file.Object(4, "(an update)"); + var nine = file.Position; + file.Raw(Invariant($"9 0{new string(' ', 3000)}obj\n{older}\nendobj\n")); + var newer = file.Position; + file.Raw(Invariant( + $"xref\n4 1\n{four:D10} 00000 n \n9 1\n{nine:D10} 00000 n \ntrailer\n<< /Size 10 /Root 1 0 R /Prev 9 0 R >>\nstartxref\n{newer}\n%%EOF\n")); + + using (var whole = PdfDocument.Open(file.Bytes)) + { + whole.WasRepaired.Should().BeFalse(); + whole.Reader.Structure.Sections.Should().HaveCount(2); + } + + using var document = PdfDocument.Open(file.Bytes, ObjectGuard); + + var section = document.Reader.Structure.Sections.Should().ContainSingle().Which; + section.Offset.Should().Be(newer); + section.ChainHiddenByGuard.Should().Be("its /Prev is the reference 9 0 R, which one of the reader's limits cut"); + document.Diagnostics.Where(entry => entry.Code == PdfDiagnosticCodes.LimitObject).Select(entry => entry.ToString()).Should().Equal( + Invariant($"Warning limit.object at {nine}: Object 9 runs past 1 KB, its stream data aside; only what lies within it was read. Raise PdfReaderLimits.MaxObjectLength to read past it.")); + var report = new PdfValidator().Validate(document); + report.Contains(PdfValidationRuleIds.XRefSectionNotFound).Should().BeFalse(); + report.Findings.Where(finding => finding.RuleId == PdfValidationRuleIds.XRefCheckedInPart).Select(finding => finding.Message).Should().Equal( + Invariant($"The cross-reference section at offset {newer} names older sections, but its /Prev is the reference 9 0 R, which one of the reader's limits cut: they were not checked."), + Invariant($"Reading the header of object 9 where its entry places it, at offset {nine}, or near it, the reader reached PdfReaderLimits.MaxObjectLength: the entry was not checked. Raising the limit lets it be.")); + + var throwing = () => PdfDocument.Open(file.Bytes, ObjectGuard with { ThrowOnLimit = true }); + throwing.Should().Throw().Which.LimitName.Should().Be(nameof(PdfReaderLimits.MaxObjectLength)); + } + [Fact] public void A_cross_reference_stream_s_prev_a_guard_cut_ends_the_chain_as_the_guard_s() { @@ -3350,6 +3613,8 @@ public void A_stream_whose_value_a_guard_cut_where_the_chain_names_a_section_is_ [InlineData("<< /Type /XRef /W [1 9223372036854775807 5 0 R] /Size 8 /Length 56 >>\nstream\n{56}\nendstream", "0", "/Prev", "holds at most 0 of the 8 rows", false)] [InlineData("<< /Type /XRef /W [4611686018427387904 4611686018427387904 5 0 R] /Size 8 /Length 56 >>\nstream\n{56}\nendstream", "0", "/Prev", "holds at most 0 of the 8 rows", false)] [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Filter /FlateDecode /Length {length} >>\nstream\n{flate}\nendstream", "[1 4 2]", "/Prev", "its data, 5 bytes decoded, holds at most 5 of the 8 rows", true)] + [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Filter [/FlateDecode] /DecodeParms [<< /Predictor 1 >>] /Length {length} >>\nstream\n{flate}\nendstream", "[1 4 2]", "/Prev", "its data, 5 bytes decoded, holds at most 5 of the 8 rows", true)] + [InlineData("<< /Type /XRef /W 5 0 R /Size 8 /Filter /FlateDecode /DecodeParms << /Predictor 1 >> /Length {length} >>\nstream\n{flate}\nendstream", "[1 4 2]", "/Prev", "its data, 5 bytes decoded, holds at most 5 of the 8 rows", true)] public void A_section_whose_data_holds_fewer_rows_than_it_declares_whatever_a_value_a_guard_cut_holds_is_the_file_s_to_lack( string six, string five, string naming, string fault, bool hidesRows) { @@ -3358,7 +3623,8 @@ public void A_section_whose_data_holds_fewer_rows_than_it_declares_whatever_a_va // read whole, it lacks rows, the file's, its trailer merged, the chain going on, and the index rebuilt. It was taken // for the guard's: the chain ended, nothing was rebuilt and the catalog was lost (#263); then for a section that // cannot be read, whose trailer was lost, and which, where the chain names a place a few bytes off, was passed over - // for an older revision's stream nearby. Named by /XRefStm it is read as named. + // for an older revision's stream nearby. Named by /XRefStm it is read as named. Filters written direct are a name or + // an array of names, beside parameters written as a dictionary, or an array of them, that holds no reference. using var output = new MemoryStream(); using (var zlib = new ZLibStream(output, CompressionLevel.NoCompression, leaveOpen: true)) @@ -3424,6 +3690,123 @@ public void A_section_whose_data_holds_fewer_rows_than_it_declares_whatever_a_va : wholeRules); } + [Theory] + [InlineData("/Filter [8 0 R]", "/FlateDecode")] + [InlineData("/Filter [/FlateDecode] /DecodeParms [<< /Predictor 8 0 R >>]", "1")] + public void Data_whose_filters_a_reference_gives_says_nothing_of_the_rows_a_section_beside_a_value_a_guard_cut_lacks( + string filters, string eight) + { + // As above, object 6's /W names object 5, which MaxObjectLength cuts, and its data, whose length it gives, decodes to + // 5 bytes: fewer than the 8 rows its /Size declares, at a byte a row. Read whole, it lacks rows. But an element of its + // /Filter array, or a value of the parameters its /DecodeParms array gives its one step, is a reference to object 8, + // which the newer table places — as a PDF 1.x file may write it, only ISO 32000-2 making it direct (7.5.8.2, + // pdf-issues #246) —, and only data whose filters are written direct, no reference in them, is decoded to judge the + // rows beside a cut (ADR 34). The section is the guard's, the chain ending there and nothing rebuilt. + var file = NamedBesideCut( + "<< /Type /XRef /W 5 0 R /Size 8 " + filters + " /Length {length} >>\nstream\n{data}\nendstream", eight, flates: 1, out var sixAt); + + using (var whole = PdfDocument.Open(file)) + { + whole.WasRepaired.Should().BeTrue(); + whole.Reader.Structure.Sections[^1].Incomplete.Should().BeTrue(); + } + + using var document = PdfDocument.Open(file, ObjectGuard); + + var section = document.Reader.Structure.Sections[^1]; + section.Offset.Should().Be(sixAt); + section.State.Should().Be(XRefSectionState.Malformed); + section.CutByLimit.Should().BeTrue(); + section.Incomplete.Should().BeFalse(); + section.Fault.Should().Be("its /W is the reference 5 0 R, which one of the reader's limits cut"); + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(PdfDiagnosticCodes.LimitObject); + } + + [Theory] + [InlineData(nameof(PdfReaderLimits.MaxFilterChainLength), PdfDiagnosticCodes.LimitFilterChain)] + [InlineData(nameof(PdfReaderLimits.MaxDecodedStreamLength), PdfDiagnosticCodes.LimitDecodedStream)] + public void Data_a_guard_on_decoding_stops_says_nothing_of_the_rows_a_section_beside_a_value_a_guard_cut_lacks( + string limit, string code) + { + // As above, but object 6's /Size declares 40 rows, and its filters, written direct, are two Flate steps, the second of + // which a MaxFilterChainLength of 1 leaves undone, or one, whose 5 bytes a MaxDecodedStreamLength of 4 cuts: what the + // guard on decoding leaves is fewer bytes than the 40 rows, at the one byte a row the cut /W leaves at the fewest. Read + // whole, the section lacks rows; under the guard on decoding alone, the rows past what it decoded are the guard's. + // Beside the value MaxObjectLength cuts too, how many rows the data holds is the guard's to hide: the section is the + // guard's, the chain ending there, nothing rebuilt, and each guard reported. + var twoSteps = limit == nameof(PdfReaderLimits.MaxFilterChainLength); + var file = NamedBesideCut( + "<< /Type /XRef /W 5 0 R /Size 40 /Filter " + (twoSteps ? "[/FlateDecode /FlateDecode]" : "/FlateDecode") + + " /Length {length} >>\nstream\n{data}\nendstream", + "(eight)", + twoSteps ? 2 : 1, + out var sixAt); + var decoding = twoSteps ? PdfReaderLimits.Default with { MaxFilterChainLength = 1 } : PdfReaderLimits.Default with { MaxDecodedStreamLength = 4 }; + + using (var whole = PdfDocument.Open(file)) + { + whole.WasRepaired.Should().BeTrue(); + whole.Reader.Structure.Sections[^1].Incomplete.Should().BeTrue(); + } + + using (var decoded = PdfDocument.Open(file, new PdfReaderOptions { Limits = decoding })) + { + decoded.WasRepaired.Should().BeFalse(); + decoded.Reader.Structure.Sections[^1].CutByLimit.Should().BeTrue(); + decoded.Reader.Structure.Sections[^1].Incomplete.Should().BeFalse(); + } + + using var document = PdfDocument.Open(file, new PdfReaderOptions { Limits = decoding with { MaxObjectLength = 1024 } }); + + var section = document.Reader.Structure.Sections[^1]; + section.Offset.Should().Be(sixAt); + section.State.Should().Be(XRefSectionState.Malformed); + section.CutByLimit.Should().BeTrue(); + section.Incomplete.Should().BeFalse(); + section.Fault.Should().Be("its /W is the reference 5 0 R, which one of the reader's limits cut"); + document.WasRepaired.Should().BeFalse(); + document.Diagnostics.Select(entry => entry.Code).Should().Equal(code, PdfDiagnosticCodes.LimitObject); + } + + /// + /// Builds the file of the tests above: object 6, a stream whose dictionary and data gives — + /// {length} standing for the data's length, and {data} for 5 bytes of 0 under + /// steps of Flate kept uncompressed —, which stream 7 indexes; object 5, [1 4 2] past MaxObjectLength; and a newer + /// table that places objects 5 and 8, which holds , and names object 6 through its /Prev. + /// + private static byte[] NamedBesideCut(string six, string eight, int flates, out long sixAt) + { + var data = new byte[5]; + + for (var i = 0; i < flates; i++) + { + using var output = new MemoryStream(); + + using (var zlib = new ZLibStream(output, CompressionLevel.NoCompression, leaveOpen: true)) + { + zlib.Write(data); + } + + data = output.ToArray(); + } + + var built = ChainFiles.UnderAnUpdate( + "/Type /XRef /Size 8 /W [1 4 2] /Length {length}", + "[1 4 2]" + new string(' ', 1100), + compressed: false, + predicted: false, + placed: true, + six: six + .Replace("{data}", Encoding.Latin1.GetString(data), StringComparison.Ordinal) + .Replace("{length}", Invariant($"{data.Length}"), StringComparison.Ordinal), + placedToo: new Dictionary { [8] = eight }); + var text = Encoding.Latin1.GetString(built); + var stream = text.IndexOf("\n7 0 obj", StringComparison.Ordinal) + 1; + sixAt = text.IndexOf("\n6 0 obj", StringComparison.Ordinal) + 1; + return Encoding.Latin1.GetBytes(text.Replace(Invariant($"/Prev {stream} "), Invariant($"/Prev {sixAt} "), StringComparison.Ordinal)); + } + [Theory] [InlineData("/Prev")] [InlineData("/XRefStm")] diff --git a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs index e2763586..f93209ac 100644 --- a/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs +++ b/tests/AdCodicem.Pdf.Tests/WindowEdgeTests.cs @@ -259,6 +259,59 @@ public void An_endobj_a_regular_byte_goes_on_is_missing_wherever_the_window_edge failures.Should().BeEmpty(); } + [Fact] + public void An_endobj_the_window_s_edge_reaches_ends_the_object_where_a_source_cut_short_after_opening_gives_no_byte_more() + { + // Object 6's endobj ends right at the edge of the 8 KB window it is read through, and an "x" follows it: "endobjx" is + // no endobj, and the object lacks it. Once the document is open, the file is cut short right where the endobj ends, + // as another process truncates it: the source gives no byte past the edge, and the file says the token ends there, + // as where the file ends. The object reads whole, and nothing is reported (#174). + const string Head = "6 0 obj\n("; + const string Tail = ")\nendobj"; + var file = new HandWrittenFile(); + var rows = new Dictionary + { + [1] = file.Object(1, "<< /Type /Catalog /Pages 2 0 R >>"), + [2] = file.Object(2, "<< /Type /Pages /Kids [] /Count 0 >>"), + [6] = file.Position, + }; + file.Raw(Head + new string('a', Window - Head.Length - Tail.Length) + Tail + "x\n"); + rows[7] = file.Object(7, "(seven)"); + file.Table(8, rows, "/Size 8 /Root 1 0 R"); + var bytes = file.Bytes; + + using var whole = PdfDocument.Open(bytes); + using var source = new CutShortSource(bytes); + using var document = PdfDocument.Open(source, options: null, ownsSource: false); + source.CutAt((int)rows[6] + Window); + + whole.GetObject(new PdfObjectId(6)); + whole.Reader.IsEndObjMissing(6, out _).Should().BeTrue(); + document.GetObject(new PdfObjectId(6)).Should().BeOfType().Which.Length.Should().Be(Window - Head.Length - Tail.Length); + document.Reader.IsEndObjMissing(6, out _).Should().BeFalse(); + document.Diagnostics.Should().BeEmpty(); + } + + [Fact] + public void The_file_answers_for_a_window_s_edge_outside_it_without_asking_its_source() + { + // The parser asks the file what follows a token, or a stream keyword, its window's edge reaches, which lies inside the + // file (#174, #224). Asked of an offset outside it, before its start or at its end, the file answers without asking + // a source that refuses any read outside its data: no byte there goes on a token, nor is an end-of-line there skipped + // before a stream's data, which starts where the keyword ends. + var bytes = new TestPdfBuilder() + .WithObject(1, "<< /Type /Catalog /Pages 2 0 R >>") + .WithObject(2, "<< /Type /Pages /Kids [] /Count 0 >>") + .BuildClassic(rootNumber: 1); + using var document = PdfDocument.Open(new StrictCountingSource(bytes), options: null, ownsSource: true); + var provider = (IPdfStreamDataProvider)document.Reader; + + provider.TokenEndsAt(-1).Should().BeTrue(); + provider.TokenEndsAt(bytes.Length).Should().BeTrue(); + provider.DataStart(-1).Should().Be(-1); + provider.DataStart(bytes.Length).Should().Be(bytes.Length); + } + [Fact] public void A_cross_reference_stream_reads_whole_when_its_guard_s_window_ends_on_its_keyword() { From 9521a475aba60e67f9539d0f5968d74ed943c34b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 10:36:12 +0000 Subject: [PATCH 144/146] test: say what the probe makes of a token past xref, and hold the section it records The test of a token the cross-reference probe met cut said the probe saw "xref" and the table's own window refused "xrefs". The probe now reads on past a token its edge reaches, so it reads "xrefs" whole, finds neither the keyword nor an object, and the table never sees it. The comment says so, and the test now holds the section the probe records: not found, the fault naming neither. Where MaxTrailerLength stops the probe right after "xref", the table's window refuses the token under another rule, which is filed. Refs #174, #295 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- tests/AdCodicem.Pdf.Tests/HostileInputTests.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs b/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs index 0351b6c7..0838e9b2 100644 --- a/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs +++ b/tests/AdCodicem.Pdf.Tests/HostileInputTests.cs @@ -1762,8 +1762,10 @@ public void Reports_chains_that_run_too_deep_once() [Fact] public void Does_not_take_a_keyword_the_probe_saw_cut_for_a_cross_reference_table() { - // The section is probed through a few bytes, which spaces and "xref" fill: the probe sees "xref", but the - // token is "xrefs". The table's own window reads it whole and refuses it, and the index is rebuilt. + // The section is probed through a few bytes, which spaces and "xref" fill, but the token is "xrefs". The probe reads + // on past a token its edge reaches, finds no keyword there, nor an object, and the index is rebuilt (#174). Where + // MaxTrailerLength stops the probe right after "xref", the table's own window reads the token whole and refuses it + // under another rule (#295). var written = new TestPdfBuilder() .WithObject(1, "<< /Type /Catalog /Pages 2 0 R >>") .WithObject(2, "<< /Type /Pages /Kids [] /Count 0 >>") @@ -1774,6 +1776,8 @@ public void Does_not_take_a_keyword_the_probe_saw_cut_for_a_cross_reference_tabl using var document = PdfDocument.Open(bytes); document.WasRepaired.Should().BeTrue(); + document.Reader.Structure.Sections[0].State.Should().Be(XRefSectionState.NotFound); + document.Reader.Structure.Sections[0].Fault.Should().Be("holds neither the xref keyword nor an object"); document.Catalog.Required().IsOfType(PdfName.Catalog).Should().BeTrue(); } From b9df53761afb00278fbb814134fa73eb0a05dac7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 10:40:17 +0000 Subject: [PATCH 145/146] docs: record batch 16 in status.md The journal of 2026-10-25 records the batch: - the issues' shapes measured and the eleven questions settled with the maintainer; - the thirteen adversarial reviews and the decisions they asked for; - the coverage pass and the faults it filed; - what the batch did for its six issues, the tests and mutants, and the checks; - the debts it filed, and the maintainer's decision on the remote corpus's hosts. The figures at a glance follow the branch: tests, coverage and the lines left with why no input reaches them, CI and the remote corpus's runs 75 to 80, and the measurements of reading every page and of a wrong /Length. Step 4's list is brought up to date, with the fourteen debts placed in the batches they join. Refs #170, #174, #220, #224, #230, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/status.md | 297 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 252 insertions(+), 45 deletions(-) diff --git a/docs/status.md b/docs/status.md index 5bafc281..6a5d4ba7 100644 --- a/docs/status.md +++ b/docs/status.md @@ -21,57 +21,69 @@ Tracking workflow mirrors from `docs/roadmap.md` (*Debt and open points*, below) the maintainer's request on 2026-10-15, merged with [#254] on 2026-10-16; the thirteenth, [#128] and [#237], with [#253], with [#258] on 2026-10-17; the fourteenth, [#129] and [#175], with [#256] and [#259], with [#265] on 2026-10-21; the fifteenth, [#160] and [#161], object stream members, with [#200], [#202], [#255], [#257], [#273], - [#274] and [#275], is in [#278], under review. Then the other batches, slices 4 to 6 ([#60] to [#62]); then the + [#274] and [#275], with [#278] on 2026-10-22; the sixteenth, [#174] and [#170], stream data, with [#220], [#224], + [#230] and [#263], is in [#297], under review. Then the other batches, slices 4 to 6 ([#60] to [#62]); then the milestone's adversarial review, slice 7 ([#137], ADR 46), once every other issue filed under M02 is closed. The debts come from the reader's own work, the threat model's first version ([#135], `docs/threat-model.md`), M01's review after the fact ([#136], `docs/reviews/M01.md`), the planning of step 4 ([#199] to [#202]), its second batch ([#207] to [#210]), its third ([#212]), its fifth ([#215], [#216]), its sixth ([#218] to [#224]), its eighth ([#229], [#230]), its ninth ([#237]), its tenth ([#241], [#247]), its twelfth ([#250]), slice 4's blockers ([#253]), its thirteenth - ([#255] to [#257]), its fourteenth ([#259] to [#264]) and its fifteenth ([#273] to [#277]); [#123] moved to M20. + ([#255] to [#257]), its fourteenth ([#259] to [#264]), its fifteenth ([#273] to [#277]) and its sixteenth ([#279] to + [#283], [#287] to [#290], [#292] to [#296]); [#123] moved to M20. - **User documentation**: organized along Diátaxis since 2026-09-30 ([ADR 47](adr/0047-the-user-documentation-follows-diataxis.md), [#148]), on `main`: a tutorial held to its sample by a test, four how-to guides, four reference pages — the validation rules among them, moved from the project documents — with the API reference under them, and four explanations. Outside any milestone. - **Last milestone closed**: **M01 — Object model and tolerant reading** -- **Tests**: 4,689 unit on [#278]'s branch (19 skipped: 2 by design, the theory over the remote corpus's streams whose +- **Tests**: 5,332 unit on [#297]'s branch (20 skipped: 2 by design, the theory over the remote corpus's streams whose length is wrong, the fifteen remote documents of the theory over the corpus's filter chains, which have no document - without them, and the test held to PDFBox's remote report) + 1,495 integration (skipped without Docker) + 23 for the - remote corpus's fetcher + 43 for the roadmap's mirror on GitHub. With the remote documents fetched, on [#278]'s - branch: 7,060 unit with the 233 this container holds, 3 skipped — the laziness test on the two documents recorded as - unsupported until [#47], which every other test holds to their expectations, and the private manifest this container - lacks —; 3,688 integration against qpdf in its container, 3,682 passed and 6 skipped where qpdf cannot walk a damaged - document's pages; and, with all 242, `Remote corpus` run 74's 7,135 acceptance tests and 3,760 referee checks, on - the code [#278] ends with. + without them, the test held to PDFBox's remote report, and iPRES's stream that lost its `endstream`) + 1,495 + integration (skipped without Docker) + 23 for the remote corpus's fetcher + 43 for the roadmap's mirror on GitHub. + With the remote documents fetched, on [#297]'s branch: 7,704 unit with the 233 this container holds, 3 skipped — the + laziness test on the two documents recorded as unsupported until [#47], which every other test holds to their + expectations, and the private manifest this container lacks —, the same under the runtime Microsoft ships; 3,688 + integration against qpdf in its container, 3,682 passed and 6 skipped where qpdf cannot walk a damaged document's + pages; and, with all 242, `Remote corpus` run 80's 7,742 acceptance tests and 3,760 referee checks, on the code + [#297] ends with. - **Coverage**: on the committed corpus, as Codecov counts it (a line with an untaken branch is partial, the generated - Arlington tables left out), 99.51 % of `src/` — 7,273 of 7,309 lines on [#278]'s branch, 380 of its patch's 383 - measurable lines, two partial and one missed. The 36 left are those the rule of 2026-09-29 leaves (`CLAUDE.md`, - *Coverage*): `main`'s 34 since [#265], at the line numbers [#278] gives them, and two more. They are members that are - private, or of a private type, which no input reaches — eight lines of `ArlingtonWalk` (194, 925, 1029, 1039, 1040, - 1060, 1062, 1152) and three of a defensive branch of `PdfLexer` (161, 164, 165) —, a `?.` on an index never null - where it is read and a switch's default arm (`CrossReferenceProbe` 108 and 269, `PageTreePageOrphanedRule` 51, - `RootInvalidRule` 111 and 120), a line the compiler puts after a call that never returns (`PdfFileReader` 2218), - three branches the compiler adds that no input takes (`FileQuote` 145, an interpolation's buffer too small; - `PdfObjectParser` 875, a concatenation's null check; `PdfObjectParser` 795, the switch over a `/Length`'s presence, - whose every arm is taken); `EndStreamAfter`'s two defensive arms (`PdfFileReader` 2186, 2190, journal of batch 6), - the `HasValue` checks the compiler adds on a `long?` (3854, batch 5) and a defensive arm of the real's expansion - (`PdfReal` 66, batch 5), one of the six ways out of a cross-reference stream row's `switch` (3987) and `Repair`'s - guard against a second rebuild (5229, 5231); `KeepObjectStream`'s release of the contents it replaces (5184, 5186, - batch 13), the second of which batch 15 rewrote to weigh a header's tables with the data, and so the patch's one - missed line. Batch 14's five: the switch over a header's place (`CrossReferenceProbe` 167), whose target past the + Arlington tables left out), 99.42 % of `src/` — 8,124 of 8,171 lines on [#297]'s branch, 1,284 of its patch's 1,293 + measurable lines, nine partial and none missed. The 47 left are those the rule of 2026-09-29 leaves (`CLAUDE.md`, + *Coverage*), and five that only faults filed reach. `main`'s 36 since [#278] are 32 now, at the line numbers [#297] + gives them. They are members that are private, or of a private type, which no input reaches — eight lines of + `ArlingtonWalk` (194, 925, 1029, 1039, 1040, 1060, 1062, 1152) and three of a defensive branch of `PdfLexer` (161, 164, + 165) —, a `?.` on an index never null where it is read and a switch's default arm (`CrossReferenceProbe` 108 and 269, + `PageTreePageOrphanedRule` 51, `RootInvalidRule` 111 and 120), a line the compiler puts after a call that never + returns (`PdfFileReader` 2580), two branches the compiler adds that no input takes (`FileQuote` 145, an interpolation's + buffer too small; `PdfObjectParser` 1218, a concatenation's null check); a defensive arm of the real's expansion + (`PdfReal` 66, batch 5), one of the six ways out of a cross-reference stream row's `switch` (5187) and `Repair`'s guard + against a second rebuild (6458, 6460); `KeepObjectStream`'s release of the contents it replaces (6413, 6415, batch + 13). Batch 14's five: the switch over a header's place (`CrossReferenceProbe` 167), whose target past the enumeration's values the compiler adds and no input takes; the one over a kid's presence (`PageTreeWalk` 345), whose default the batch's arm for a cut object leaves to `Deferred`, which `GetPresence` never answers; the test that a judged catalog's `/Pages` is a dictionary or a stream (`ObjectGraph` 177), which the walk judged a root there, so is one; and two defensive lines of a rebuild: `FinishRebuild`'s `??=` when reading `/Root` reaches a guard - (`PdfFileReader` 5344), which always finds one held already — a `/Root` a guard cuts is a regular object, or a member + (`PdfFileReader` 6573), which always finds one held already — a `/Root` a guard cuts is a regular object, or a member of an object stream, that the rebuild read as it took in object streams, reaching the guard first —, and - `KeepMembersHiddenByGuard`'s position of a stream the rebuilt index lacks (5602), when every stream it marks hidden - was read from that index. Batch 15 adds two, in the probe's judging of the members only a rebuild places: the arms - that pass over an object stream the rebuilt index does not hold as a regular object, or whose header the reader - cannot read as one (`CrossReferenceProbe` 431), when the rebuild placed the members there for having found that - stream and read its header; and a `?.` on the chain's index (445), which a chain a guard stopped has read. A private - method nothing had called since the batch's reading at the chain's place, `TryParseAt`, which coverage found, is gone. - `codecov.yml` asks 95 % of each patch, and lets the project drop by half a point at most; the aim is 100 %. -- **CI**: green on `main` at `6a5aad3` (CI run 641), [#265]'s last commit. `Remote corpus` passed on every document in + `KeepMembersHiddenByGuard`'s position of a stream the rebuilt index lacks (6831), when every stream it marks hidden + was read from that index. Batch 15's two, in the probe's judging of the members only a rebuild places: the arms that + pass over an object stream the rebuilt index does not hold as a regular object, or whose header the reader cannot + read as one (`CrossReferenceProbe` 431), when the rebuild placed the members there for having found that stream and + read its header; and a `?.` on the chain's index (445), which a chain a guard stopped has read. Batch 16 takes three + off — `EndStreamAfter`'s two defensive arms, which a document whose searches are spent now takes, and the switch over + a `/Length`'s presence —, brings a fourth into its patch, the `HasValue` checks the compiler adds on a `long?` + (`PdfFileReader` 5040), and leaves fifteen. In its patch, nine no input takes: a `?.` on the set of what the chain + read, never null where it is forgotten (2517), and on a cross-reference stream's `/W` element, `/Size` and `/Index` + pair, which the parser never leaves null there (4575, 4612, 4628); those `HasValue` checks, on values already tested + (5040); a deferred length's search, always memoized by the parse before (4325), and its value, never null (4503), + defensive arms of private members; the operand of `ChecksumDisagrees` its one caller's precondition makes true + (`PdfFilterPipeline` 343); and a header's generation neither cut nor an integer, which the tests before it rule out + (`PdfObjectParser` 1570). Outside it, the dispatch the compiler builds over `SkipContainer`'s token kinds, to which + the batch's arm for a header adds a leaf no kind reaches (`PdfObjectParser` 1296); and five lines that inputs still + reach, but only to show a fault filed rather than pinned: a classic table's first token the probe's bound cut + (`PdfFileReader` 3349, 3351, 3352, [#295]), and `MarkCutByGuard`, which a `stream` a guard's edge parts reaches + (5878, 5880, [#294]). `codecov.yml` asks 95 % of each patch, and lets the project drop by half a point at most; the + aim is 100 %. +- **CI**: green on `main` at `a76c9da` (CI run 678), [#278]'s last commit. `Remote corpus` passed on every document in the nightly run 18, on `main` at `8c54490`, and in runs 19 and 20, on [#203]'s and [#211]'s branches, each rerun once after web.archive.org refused a document ([#204]), in run 21, on [#213]'s branch, at its first attempt, and in run 22, on [#214]'s branch, rerun once for the same refusal. The nightly run 23, on `main`, failed only because @@ -114,7 +126,13 @@ Tracking workflow mirrors from `docs/roadmap.md` (*Debt and open points*, below) expectation. Run 71, on `48c7172`, passed whole at its first attempt, and so did run 72, on `e69756b`. Run 73, on `0de8727`, the same tree under a corrected commit message, lost ECan's scan at its first attempt, web.archive.org refusing the connection ([#204]), every test over the 241 others passing, and passed whole at its one rerun. Run 74, - on the code [#278] ends with, passed whole at its first attempt: 7,135 acceptance tests and 3,760 referee checks. + on the code [#278] ends with, passed whole at its first attempt: 7,135 acceptance tests and 3,760 referee checks. The + nightly run 75, on `main` at `a76c9da`, passed whole; run 77 lost ECan's scan, its host timing out in the TLS + handshake, and run 79 the 88 documents of the ipres2017 archive, its host timing out ([#204]), every test over the + others passing. On [#297]'s branch, run 76, on `8f6a085`, passed whole; run 78, on `241d701`, lost the ipres2017 + archive the same way and failed one test, whose bounds the zlib of this container set and the zlib-ng of the runtime + CI installs does not keep, which `d468352` corrected; run 80, on that commit, passed whole at its first attempt: + 7,742 acceptance tests and 3,760 referee checks. - **Corpus**: 168 committed documents, 23.0 MB — 19 generated here, 3 from Word and PDF24 on Windows, 146 third-party files under attribution-only licenses (`docs/corpus-sources.md`). Beside them, a **remote corpus** of 242 documents we may use but not redistribute, fetched at a pinned SHA-256 and size (ADR 32), @@ -164,7 +182,7 @@ Tracking workflow mirrors from `docs/roadmap.md` (*Debt and open points*, below) | Operation | Document | Time | Allocated | |---|---|---|---| | Indexing | synthetic, 1000 pages, ~4 MB | 280 µs | 122 KB | -| Indexing, then reading every page | synthetic, 1000 pages, ~4 MB | 4.8 ms | 5.7 MB | +| Indexing, then reading every page | synthetic, 1000 pages, ~4 MB | 4.2 ms | 1.8 MB | | Indexing and walking the page tree | real ReportLab document, 1000 pages | — | 3.2 MB | | Indexing a classic table (xUnit, after a first reading) | generated, 300,000 objects | — | 16.9 MB | | Indexing a cross-reference stream whose rows Flate stores (xUnit, after a first reading) | generated, 300,000 objects | — | 29.5 MB | @@ -182,8 +200,8 @@ Tracking workflow mirrors from `docs/roadmap.md` (*Debt and open points*, below) | Decoding the same stream under a wrong checksum, kept whole | 4 MB decoded | 12.1 ms | 13.1 MB | | Decoding a stream that turns corrupt halfway, the 2 MB before the fault kept | 4 MB of content | 8.1 ms | 6.6 MB | | Decoding a chain of ASCII85 and Flate | 4 MB decoded | 10.4 ms | 13.7 MB | -| Indexing, then parsing every page's content stream, each past the reader's first window | synthetic, 1000 pages of 16 KB | 3.9 ms from memory, 7.1 ms from a file | 2.0 MB | -| The same, each stream declared 2 bytes too long and searched for its `endstream` | synthetic, 1000 pages of 16 KB | 7.3 ms from memory, 12.5 ms from a file | 2.8 MB | +| Indexing, then parsing every page's content stream, each past the reader's first window | synthetic, 1000 pages of 16 KB | 4.0 to 4.6 ms from memory, 7.2 to 7.4 ms from a file | 1.75 MB | +| The same, each stream declared 2 bytes too long and searched for its `endstream` | synthetic, 1000 pages of 16 KB | 10.6 to 11.3 ms from memory, 14.9 to 16.4 ms from a file | 2.71 MB | | Parsing one object, a medium job | Word's page dictionary | 6.0 µs | 4.6 KB | The gap between the first two rows is the library's promise: opening a document does not read its content. @@ -224,8 +242,10 @@ runs' spread. Indexing costs the map, 52 to 63 bytes an object, whatever the objects weigh, since a section makes room for its rows before reading them ([#164]); a map a rebuild grows costs 100 to 210, by where the count falls against its growth steps. A cross-reference stream adds 7 to about 42 bytes an object for its rows, and the object stream holding the catalog, -when one does, what it holds. The first two rows are batch 11's (ShortRun, twice): 394 KB and 6.1 MB on `main` in the -same runs, 304 to 324 µs and 5.5 to 6.5 ms. The third row is asserted as a budget in CI +when one does, what it holds. The first row is batch 11's (ShortRun, twice): 394 KB on `main` in the same runs, 304 to 324 +µs. The second is batch 16's ([#297], ShortRun, twice, alternated): no stream keeps its raw bytes since, each read from +its source when it is asked for, so reading every page takes 4.1 to 4.3 ms and 1.79 MB, against 6.3 to 7.1 ms and +5.82 MB on `main`. The third row is asserted as a budget in CI (`CorpusReadingTests`), so an allocation regression fails the build; it was 2.4 MB when M01 closed, and M01's review measured 3.2 MB on 2026-10-01, under the 4 MB budget. The three generated rows are held to 17.7 MB, 31.0 MB and 32.1 MB, 5 % over their figures (`DocumentReaderTests`, [#193], [#164]); growing the map allocated 31.1 MB, 43.1 MB and 33.2 MB; MB here are 2^20 bytes. A stream that ran out is @@ -253,7 +273,10 @@ read cuts of a header into the next, and the first read sized from what the pars beside it, 7.1 → 7.0 ms and 11.4 → 11.1 ms measured the same way. Data made of near-headers costs the search more than data that holds none: 16 MB of ` obj` over and over take about 63 ms to search to the end, of `0 0 obj ` 44 ms, of letters 10 ms — some 4 ms a megabyte searched at worst, and, the budget letting a document's searches read about -five times the file, some 20 ms a megabyte of file. Inside the window the check is what it was, but a wrong length +five times the file, some 20 ms a megabyte of file. Since batch 16 ([#297], medium job, twice, alternated) a sound +stream costs what it did, and a wrong length about 2 µs and 120 to 300 bytes more: one search inside the window or +past it, memoized with what follows the `endstream`, up to the next object's header or the next object the index +places; the last two rows were 8.5 to 9.1 ms and 12.4 to 13.6 ms, 2.59 MB, on `main` in the same runs ([#296]). Inside the window the check is what it was, but a wrong length now costs its record and its mark of one report: about 440 bytes more allocated per stream (the 4 KB streams, 2.20 → 2.64 MB), and about 180 bytes kept for the life of the document — 50,000 such streams, read with a cache of 64 objects, keep 9.1 MB more. A sound stream allocates and keeps nothing more. The 4 KB streams take 3.3 → 3.5 ms @@ -321,9 +344,10 @@ One pull request per batch, each design question put to the maintainer after mea cut — done, merged with [#265] on 2026-10-21, `Remote corpus` runs 63 to 65 green on its branch (journal of 2026-10-18). 15. [#160] and [#161], object-stream members, with [#200], [#202], [#255] and [#257], and [#273] to [#275], which the - batch filed — done, in [#278], `Remote corpus` runs 71 to 74 green on its branch, run 73 at its one rerun - (journal of 2026-10-22). - 16. [#174] and [#170], stream data, with [#220], [#224], [#230] and [#263]. + batch filed — done, merged with [#278] on 2026-10-22, `Remote corpus` runs 71 to 74 green on its branch, run 73 + at its one rerun (journal of 2026-10-22). + 16. [#174] and [#170], stream data, with [#220], [#224], [#230] and [#263] — done, in [#297], `Remote corpus` runs + 76 and 80 green on its branch (journal of 2026-10-25). 17. [#155]. 18. [#167] and [#168], with [#201] and [#277]. 19. [#188], [#197] and [#156], with [#207], [#208], [#210], [#221], [#222], [#241], [#261] and [#264]. @@ -357,7 +381,10 @@ One pull request per batch, each design question put to the maintainer after mea with [#174], [#170] and [#224], the stream data it concerns; [#264] with [#188] and [#197], the chain's reading of where its sections lie. Of the five batch 15 filed, [#273] to [#275] were paid in the batch at the maintainer's choice; [#276] goes with [#189], whose choice of the definition a rebuild serves it depends on; [#277] with [#168], - whose walk of the object streams' dependencies it concerns. + whose walk of the object streams' dependencies it concerns. Of the fourteen batch 16 filed, [#279], [#282], [#287], + [#289], [#290] and [#292] to [#295] go with [#163], the reader at its guards; [#280] and [#296] with [#181], the work + one document asks; [#281] with [#188], the chain's reading of where its sections lie; [#283] and [#288] with [#162], the + filter reports they reword. 5. Slice 4 ([#60]) in two pull requests, streams then fonts, after the decisions it waits on: the severity of a font that is not embedded, the standard 14's aliases, where text is "meant to be extractable", how the rules that need content are left out and shown so, and the tools that referee both families. @@ -379,6 +406,170 @@ not a fault of the file: the rules on it report at most, as information, that it ## Journal +### 2026-10-25 — Batch 16: where a stream's data ends, and what a guard's cut leaves ([#170], [#174], [#220], [#224], [#230], [#263]) +- **The question.** Measured on `a76c9da` by seven agents and a completeness critic. [#174]: a `/Length` no `endstream` + confirmed was searched for within the parser's window, then past it, a window grown for it counting toward + `MaxObjectLength` — a `/Length 2147483647` before 21 MB read 21.6 MB and reported `limit.object`, fifty such objects + in 17 MB read 1.08 GB, 63 times the file —, and a stream that lost its `endstream` took 49 bytes across the next + object's header; the corpus's 35,716 confirmed streams leave 0 to 2 bytes before `endstream`. [#170]: every stream + kept its raw bytes — 400 streams over one megabyte held 403 MB —, a `/Length` past what an array holds threw + `OutOfMemoryException`, and 50 MB no filter decodes came back whole under a 1 MB `MaxDecodedStreamLength`. [#220]: a + cross-reference stream whose `/Length` only its own rows place was read within the trailer's 64 KB window — 70 KB of + rows reached `limit.trailer` —, and rows that spell `endstream` served a superseded object. [#224]: a `/Length` a guard + cut read as "holds null", or as the integer the cut parted. [#263]: a `/Filter` or `/DecodeParms` naming an object a + guard cut made an object stream unreadable and set off a catalog search and Errors, and a cut value of a + cross-reference stream rebuilt the index; `LimitSince` blamed a guard reached while a filter value loaded on the + stream. [#230]: an object that lost its `]` or `>>` took the next object's entries. Eleven questions followed. +- **Settled with the maintainer**, all eleven as recommended (2026-10-22 and 23): + - a `/Length` is confirmed by an `endstream` past any white space and comments, read on past the window and counted + in `EndStreamSearchPasses`; `MaxEndStreamGap` removed (Q1); + - one search for every unconfirmed length, in the bytes held then the file, up to the nearer of the first header and + the next object the index places, memoized; stream data never counts toward `MaxObjectLength` (Q2); + - with no `endstream` before that stop, the object's `endobj` plays its part, each case reported as the file wrote it + (Q3); + - the next object's header ends every container left open, with `HeaderText`'s rule, in files and trailers, not in + object-stream members nor at top level (Q4); + - nothing of a stream's raw data is kept, a source held in memory lending it; the overlap's reads on a file source + filed as debt (Q5); + - a new guard, `MaxStreamLength`, 256 MB, `limit.stream-length`, bounding what is read of one stream's encoded data + (Q6); + - a deferred cross-reference stream's data searched past the window, no header stopping it, a carriage return before + the end-of-line kept; `MaxTrailerLength` bounds the dictionary alone (Q7); its data settled before its rows merge, + taken again where its own row for the `/Length` gives a length an `endstream` confirms (Q8); + - a `/Length` keyed on the object it leads to: one a guard cut or left out of the index is the guard's, the integer + read before the cut only a place to look at (Q9); + - a cross-reference stream's value that leads to an object a guard cut makes the section the guard's, the chain + ending there, nothing rebuilt (Q10); + - `LimitSince`, the trailer that lost its `>>`, and `FileStreamData`'s second copy paid here; the parted tokens and + the overlap's reads filed (Q11). +- **Reviewed** thirteen times, each finding put to a refuter in its own worktree, 325 findings in all. Every major was + fixed in the batch, with a test that failed before its fix; what was left is filed below. + - The first round, in two passes over six lenses, gave 30 findings, 6 major, then 22. Decided, as recommended: a + comment counts as white space after a declared length only past white space (F6), and after the object's `endobj` + (F5); a missing `endstream` under a `/Length` left out of the index reported as under one cut (F8); a deferred + cross-reference `/Length` cut with no `endstream` the guard's (G3, X1); `limit.stream-length` reported to a + caller's diagnostics on each decode (G4); `HasImageFilter` reading what `Decode` reads (G2). + - The second, 22 findings, two settled as the first round's rules; the third, 20. Decided after the third: a header + the data spells, under a hidden `/Length`, passed for the next object the index places (R3-18), which the fourth + replaced. + - The fourth, 25 findings, 11 major. Decided: no second pass — data a spelled header stops under a hidden `/Length`, + with no `endobj` of the object's own before it, is the guard's (R4-1); a reference the guard's edge parts after + `N G ` no missing `endobj` (R4-23); a member value ending at `MaxObjectLength` filed (R4-2, [#282]); a hybrid stream's + own `/Prev` judged as written (R4-15). + - The fifth, 24 findings, 20 kept, 5 major; the sixth, 22, 6 major. Decided after the sixth: an `endstream` under a + hidden `/Length` ends data that is the file's only where what follows the object proves it — its `endobj`, the + next object the index places, the end of the file, `xref` or `startxref` (R6-1), the chain aligned on that proof + (R6-2); a filter fault reported over data a guard cut filed (R6-3, [#283]). + - The seventh, 27 findings, 24 kept, 6 major; the eighth, 23, 20 kept, 8 major. Decided after the eighth: data an + `endstream` proven its own ends, past white space only a hidden length says is the data's or not, has its end + decoded as hidden (R8-1); the cross-reference streams the chain read place the object after a stream, rows a newer + section superseded filed (R8-2, [#287]); a place a section may have moved to is kept as the guard's only where it + declared itself one before the cut (R8-3). + - The ninth, 35 findings, 7 major; the tenth, 22, 15 kept; the eleventh, 29: the hidden end made precise — the first + filter decodes up to the run of white space and comments, what it meets before the file's —, a section lacking rows + beside a cut value the file's, and the cost of the proofs bounded to about three times the file. + - The twelfth, 30 findings, 14 major. Decided: each hidden-end case fixed as it stands, the two decodings kept (R12-1); + a section whose data lacks rows beside a cut value read as read whole (R12-2); one round more, limited to the + eleventh's and twelfth's changes, its majors fixed and the rest filed (R12-3). + - The thirteenth, 16 findings, 5 major, all fixed: a section lacking rows whose data may hold rows, read with the cut + value, is the guard's, the chain ending there, the guard reached once the chain is read and the validator saying the + rows unchecked; a deferred length's row looked for in any predicted row a spelled `endstream` parts — the first row + of a PNG predictor, a row the TIFF predictor left differenced —, a decoding guard that search reaches standing. + Its minors are tests, its nits documents; the question it raised of R12-2, and a Flate stream inflated up to three + times under a hidden end, filed ([#290]). + - The coverage of the patch, once the reviews were done, by an agent per group of lines and a skeptic per agent: of + the 39 lines the unit suite left partial or missed, 30 are held by tests through documents or direct calls, and 9 + no input takes. A second pass took the six lines outside the patch that `main` covered and the batch left: one the + compiler adds, five reached only by faults. The passes met four faults, each reproduced and filed rather than + pinned ([#292] to [#295]): one the batch made, a false warning once a crafted file has spent the searches + ([#293]), and three older ones, each of which the batch narrowed. +- **Done**, in [#297], 145 commits: + - [#174]: a `/Length` confirmed past any white space and comments, read on past the window; one memoized search for + every unconfirmed length, up to the next object's header or the next object the index places, which never grows a + window; with no `endstream`, the object's `endobj` ending the data, as the file wrote it, `stream.truncated` or + `stream.length-invalid`; a found `endstream` settling a `/Length` the file cannot hold; + - [#230]: the next object's header ends every container left open, `syntax.truncated-object` at the innermost, and a + trailer that lost its `>>` no longer takes the next update's entries; + - [#170]: no stream keeps its raw data, a source held in memory lending it; `MaxStreamLength`, a guard, 256 MB, + `limit.stream-length`, bounds what is read of one stream at once, data no filter decodes included; + - [#220]: a deferred cross-reference stream's data searched past the window, no `limit.trailer`; with no `endstream` + the section malformed; rows that spell `endstream` told from the data's end by the section's own row for the + `/Length`, in a predicted row the spelled `endstream` parts too, the data taken again to it; + - [#224]: a `/Length` that leads to an object a guard cut or left out of the index is the guard's, nothing said of it, + an `endstream` the search stops at the file's only where what follows the object proves it, and data whose end only + the hidden length could say decoded as such — what its first filter meets before the trailing white space the + file's, its running out there the guard's; + - [#263]: a filter chain ends at a value a guard cut or left out of the index, the data left encoded, an object stream + so left hidden, its members the guard's; a cross-reference stream's value so cut makes the section the guard's, the + chain ending there, the guard reached once the chain is read, but for what makes the section unreadable whatever the + value holds, or data that lacks the rows the values read declare, the file's; `LimitSince` gone; + - the chain read without loading, its values followed as far as a reference resolves, a whole reference a cut left + read through; a classic table named by `/XRefStm` reading no `/Prev` of its own; the validator leaving unjudged what + needs a chain a guard ended. + + Docs: ADR 34's classifications (`MaxStreamLength` a guard; `EndStreamSearchPasses` an internal constant, its reason + rewritten; `MaxEndStreamGap` removed; a value through a reference to what a guard cut the guard's), amended on + 2026-10-23, 24 and 25; `architecture.md`; the threat model's rows on stream ends, deferred lengths, guard-cut values, + the chain's reads, costs and the known gaps; `reference/reader-limits.md`, `reference/diagnostics.md`, + `reference/validation-rules.md`, `concepts/reader-limits.md`, `concepts/validation.md`, `concepts/diagnostics.md`, + `concepts/lazy-reading.md`, a guide to `MaxStreamLength`; M02. +- **Tests**: 643 new unit tests, 4,689 to 5,332 — with the remote documents fetched, 7,060 to 7,704 —, in + `StreamLengthTests`, `GuardCutValueTests`, `WindowEdgeTests`, `DocumentReaderTests`, `CrossReferenceRuleTests`, + `ReaderLimitsTests`, `HostileInputTests`, `FilterDamageTests`, `ParserTests`, `DiagnosticsTests` and their neighbors, + with `ChainFiles` and `HandWrittenFile` building the files. +- **Mutation**: mutants of each round's fixes, built and the tests of their classes run on them, every survivor killed by + a test written for it or shown equivalent; the last round, 16 mutants over the eleventh to thirteenth rounds' code, + 14 killed by the tests as they stood and the other two by a test written for each. +- **Tracking.** Filed under M02: [#279], the parted tokens, and [#282], a member value at `MaxObjectLength`'s bound, with + batch 27; [#280], the overlap's reads on a file source, with batch 25; [#281], a section relocated onto one the chain + read, with batch 19; [#283], a filter fault's size over data a guard cut, and [#288], a deferred stream's kept carriage + return read as a checksum byte, with batch 26; [#287], superseded rows in the proof of an `endstream`, [#289], white + space counted toward a window's bound, and [#290], the question R12-2 left and the inflation under a hidden end, with + batch 27; and, from the coverage pass, [#292], a `/Prev` naming an object whose header a guard cut, whose load + rebuilds the index once the chain is read, [#293], a deferred length reported past the chain's `endstream` once the + searches are spent, [#294], a `stream` a guard's edge parts taken for the keyword, and [#295], the cross-reference + probe at `MaxTrailerLength`'s bound, with batch 27; and, from the benchmarks, [#296], three wastes on the path of a + stream whose `/Length` is wrong, with batch 25. [#264]'s first gap, a `/Prev` through an object a guard cut, is + paid here, the issue told so; [#173], [#181], [#279] and [#282] were told of the shapes the reviews met in their + classes. +- **The remote corpus's hosts.** RADAR timed out on the ipres2017 archive in runs 78 and 79, as in run 42, 88 + documents lost at once. Decided by the maintainer on 2026-10-25 ([#204]): the remote documents a license lets the + project copy are kept, unmodified, in a repository of their own ([#291]), `tests/corpus/remote/` is cached between + runs where no pull request can restore it, and the fetcher waits as hosts ask; one pull request after this batch. +- **Checked**: + - PENDING-PERCOMMIT + - the integration suite twice on `241d701`, 3,688 tests, 3,682 passed and 6 skipped, both runs; the commits after it + change tests alone; + - the unit suite under the runtime Microsoft ships, as CI installs it, zlib-ng included, at `2055eeb`: 7,704 tests, + 3 skipped, as under the distribution's; + - the 401 documents this container holds, 168 committed and 233 remote, opened and validated on `main` and on the + branch at `241d701`, twice each, each tree giving the same lines both times: the trees differ in 76 lines, their + codes the same. `stream.truncated` names the declared length and the bytes to the end of the file in + `invoice-truncated-tail` and PDFBox's `pdfbox3949` and `pdfbox3950`, and JHOVE's `hul-138`; PDFBox's `pdfbox3947` + keeps the 1,302 bytes its object stream 417 declares, `stream.length-invalid` saying so and `filter.failed` that its + Flate stream ends before its data does, its header listing 47 objects in 3,414 decoded bytes rather than 81 from a + 32 KB block of zeros, and an Arlington finding's element moving from 182 to 192; iPRES's `t02-05-01-014` keeps the + 68 bytes its `endobj` follows; + - the documentation site, 263 pages, its check clean; + - `ReaderBenchmarks`, `ValidationBenchmarks`, `ParserBenchmarks`, `FilterBenchmarks`, `LexerBenchmarks`, + `StreamOpenBenchmarks` and `StreamLengthBenchmarks` (ShortRun), twice, `main` and the branch alternated, then + `StreamLengthBenchmarks` under a medium job, twice. Reading every page of a thousand takes 4.1 to 4.3 ms and 1.79 MB + against 6.3 to 7.1 ms and 5.82 MB, no stream keeping its raw bytes; opening allocates 450 bytes more, the reader's + new fields and five empty memos; the parser, the filters, the lexer and the validation rows allocate what they did, + to 40 bytes, opening and validating a thousand pages 7 KB less, their times within the runs' spread. A sound stream + costs what it did. A stream whose `/Length` is 2 bytes too long costs about 2 µs and 120 to 300 bytes more: 6.6 to + 7.8 ms against 5.2 to 5.8 ms and 2.71 MB against 2.41 MB for a thousand of 4 KB from memory, 10.6 to 11.3 ms against + 8.5 to 9.1 ms and 2.71 MB against 2.59 MB for 16 KB. The search memo, which a search inside the window now goes + through and whose entries grew from 48 to 80 bytes, and the index's offsets sorted once account for the bytes, to + the byte; three wastes on that path, a boxed `long?`, a read of bytes the window holds and `endobj` passes thrown + away, are filed ([#296]); + - `Remote corpus`: run 76, on `8f6a085`, passed whole. Run 78, on `241d701`, lost the 88 documents of the ipres2017 + archive, its host timing out ([#204]), and failed one test, a deferred length checked on data `MaxStreamLength` cuts + past every row, whose bounds were those of the Flate data this container's zlib writes, which the zlib-ng of the + runtime CI installs writes otherwise: `d468352` stores that data, and run 80, on it, passed whole, 242 documents, + 7,742 acceptance tests and 3,760 referee checks. +- **Next**: batch 17, [#155]. + ### 2026-10-22 — Batch 15: object stream members ([#160], [#161], [#200], [#202], [#255], [#257], [#273], [#274], [#275]) - **The question.** [#160]: a member of an object stream was parsed over the whole decoded data, with no `MaxObjectLength`, and the header's tables were allocated from `/N` before a pair was read — at `13e06bb`, as the @@ -2349,3 +2540,19 @@ Until 2026-09-27 this section was a table whose rows were numbered T01 to T40; t [#276]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/276 [#277]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/277 [#278]: https://github.com/AdCodicem/AdCodicem.Pdf/pull/278 +[#279]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/279 +[#280]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/280 +[#281]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/281 +[#282]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/282 +[#283]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/283 +[#287]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/287 +[#288]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/288 +[#289]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/289 +[#290]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/290 +[#291]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/291 +[#292]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/292 +[#293]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/293 +[#294]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/294 +[#295]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/295 +[#296]: https://github.com/AdCodicem/AdCodicem.Pdf/issues/296 +[#297]: https://github.com/AdCodicem/AdCodicem.Pdf/pull/297 From 0089cbc788e6c94cfece86b8a9b49692b3a37d6d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 13:17:43 +0000 Subject: [PATCH 146/146] docs: record that batch 16 lands squashed, and remote corpus run 81 The maintainer squashes the pull request, its 145 commits too many for a rebase. The record says so, gives the check of each commit as far as it had gone, and adds run 81 of the remote corpus. Run 81 lost two documents to web.archive.org at its first attempt and passed whole at its one rerun. Refs #170, #174, #220, #224, #230, #263 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PiFf9bygtdCAcN4va1ss8T --- docs/status.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/status.md b/docs/status.md index 6a5d4ba7..09a19c2a 100644 --- a/docs/status.md +++ b/docs/status.md @@ -132,7 +132,9 @@ Tracking workflow mirrors from `docs/roadmap.md` (*Debt and open points*, below) others passing. On [#297]'s branch, run 76, on `8f6a085`, passed whole; run 78, on `241d701`, lost the ipres2017 archive the same way and failed one test, whose bounds the zlib of this container set and the zlib-ng of the runtime CI installs does not keep, which `d468352` corrected; run 80, on that commit, passed whole at its first attempt: - 7,742 acceptance tests and 3,760 referee checks. + 7,742 acceptance tests and 3,760 referee checks. Run 81, on `b9df537`, the first form of the record [#297] ends with, + lost two documents at its first attempt, web.archive.org refusing the connection or timing out ([#204]), every test + over the 240 others passing, and passed whole at its one rerun; CI run 780 is green on that code. - **Corpus**: 168 committed documents, 23.0 MB — 19 generated here, 3 from Word and PDF24 on Windows, 146 third-party files under attribution-only licenses (`docs/corpus-sources.md`). Beside them, a **remote corpus** of 242 documents we may use but not redistribute, fetched at a pinned SHA-256 and size (ADR 32), @@ -483,7 +485,7 @@ not a fault of the file: the rules on it report at most, as information, that it compiler adds, five reached only by faults. The passes met four faults, each reproduced and filed rather than pinned ([#292] to [#295]): one the batch made, a false warning once a crafted file has spent the searches ([#293]), and three older ones, each of which the batch narrowed. -- **Done**, in [#297], 145 commits: +- **Done**, in [#297], 145 commits, squashed into one at the maintainer's choice, too many for a rebase: - [#174]: a `/Length` confirmed past any white space and comments, read on past the window; one memoized search for every unconfirmed length, up to the next object's header or the next object the index places, which never grows a window; with no `endstream`, the object's `endobj` ending the data, as the file wrote it, `stream.truncated` or @@ -537,7 +539,11 @@ not a fault of the file: the rules on it report at most, as information, that it project copy are kept, unmodified, in a repository of their own ([#291]), `tests/corpus/remote/` is cached between runs where no pull request can restore it, and the fetcher waits as hosts ask; one pull request after this batch. - **Checked**: - - PENDING-PERCOMMIT + - each commit of the branch built alone with no warning and run through the unit suite, a check still running when + this record was written: the first 30 build and pass, 7,065 to 7,297 tests with the remote documents fetched, but + for one run of the test of the validator's linear time over an object stream's members, on `d3c44d1`, a commit + that does not touch what it times, while other work loaded the machine. The squashed tree is the one CI and the + checks below ran on. commitlint finds no problem in the messages; - the integration suite twice on `241d701`, 3,688 tests, 3,682 passed and 6 skipped, both runs; the commits after it change tests alone; - the unit suite under the runtime Microsoft ships, as CI installs it, zlib-ng included, at `2055eeb`: 7,704 tests, @@ -567,7 +573,9 @@ not a fault of the file: the rules on it report at most, as information, that it archive, its host timing out ([#204]), and failed one test, a deferred length checked on data `MaxStreamLength` cuts past every row, whose bounds were those of the Flate data this container's zlib writes, which the zlib-ng of the runtime CI installs writes otherwise: `d468352` stores that data, and run 80, on it, passed whole, 242 documents, - 7,742 acceptance tests and 3,760 referee checks. + 7,742 acceptance tests and 3,760 referee checks. Run 81, on `b9df537`, the first form of this record, lost two + documents at its first attempt, web.archive.org refusing the connection or timing out ([#204]), every test over the + 240 others passing, and passed whole at its one rerun. - **Next**: batch 17, [#155]. ### 2026-10-22 — Batch 15: object stream members ([#160], [#161], [#200], [#202], [#255], [#257], [#273], [#274], [#275])