From 63736da27c018507fc0488d27ac5a131446f9686 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 21 Oct 2026 14:14:12 +0300 Subject: [PATCH 1/3] perf(xls): read records in place and decode strings a run at a time --- src/xls/biff.ts | 137 ++++++++++++++++++++++++++++++++++++++++------ src/xls/reader.ts | 72 +++++++++++++----------- test/xls.test.ts | 108 ++++++++++++++++++++++++++++++++++++ 3 files changed, 270 insertions(+), 47 deletions(-) diff --git a/src/xls/biff.ts b/src/xls/biff.ts index 763cc950..f2a0f53c 100644 --- a/src/xls/biff.ts +++ b/src/xls/biff.ts @@ -45,22 +45,74 @@ export const SID = { BOF: 0x0809, } as const -/** Parse a stream into records, recording each one's byte offset. */ -export function parseRecords(stream: Uint8Array): BiffRecord[] { - const out: BiffRecord[] = [] - const view = new DataView(stream.buffer, stream.byteOffset, stream.byteLength) - let pos = 0 - while (pos + 4 <= stream.length) { - const id = view.getUint16(pos, true) - const len = view.getUint16(pos + 2, true) - const start = pos - pos += 4 - out.push({ id, data: stream.subarray(pos, pos + len), offset: start }) - pos += len +/** + * Walks a stream's records in place, one at a time. + * + * The reader used to materialize every record before decoding the first: + * an object and a view apiece, ~155 bytes each, and a map from offset to + * index on top. A 17 MB workbook is ~960k records, so that was ~150 MB + * held for the whole read. Framing is unchanged — a record running past + * the end of the stream is cut short, and a zero id with a zero length is + * padding that ends it. + */ +export class RecordCursor { + private view: DataView + private ended = false + + constructor( + private stream: Uint8Array, + private pos = 0, + ) { + this.view = new DataView(stream.buffer, stream.byteOffset, stream.byteLength) + } + + /** Move past the record at the cursor and return its offset, or -1 at the end. */ + skip(): number { + if (this.ended || this.pos + 4 > this.stream.length) return -1 + const offset = this.pos + const id = this.view.getUint16(offset, true) + const len = this.view.getUint16(offset + 2, true) + this.pos = offset + 4 + len // A zero id past real data means padding — stop. - if (id === 0 && len === 0) break + if (id === 0 && len === 0) this.ended = true + return offset + } + + next(): BiffRecord | undefined { + const offset = this.skip() + if (offset < 0) return undefined + const len = this.view.getUint16(offset + 2, true) + return { + id: this.view.getUint16(offset, true), + data: this.stream.subarray(offset + 4, offset + 4 + len), + offset, + } + } + + /** The record `next` would return, without moving past it. */ + peek(): BiffRecord | undefined { + const { pos, ended } = this + const rec = this.next() + this.pos = pos + this.ended = ended + return rec } - return out +} + +/** + * Which of `offsets` begin a record. A BOUNDSHEET names its sheet by the + * byte offset of the sheet's BOF, and one landing inside a record — a + * damaged file — has always read as an empty sheet rather than as records + * taken out of phase. + */ +export function recordStarts(stream: Uint8Array, offsets: Iterable): Set { + const wanted = new Set(offsets) + const found = new Set() + const cursor = new RecordCursor(stream) + for (let at = cursor.skip(); at >= 0 && found.size < wanted.size; at = cursor.skip()) { + if (wanted.has(at)) found.add(at) + } + return found } /** Little-endian cursor over a record body. */ @@ -103,6 +155,43 @@ export class Reader { } } +/** The most characters handed to one `String.fromCharCode` call. */ +const CHARS_PER_CALL = 8192 + +/** + * `count` characters from `start`: a byte each when compressed (the byte + * is the code point), else UTF-16LE code units. The caller guarantees the + * bytes are there. + * + * Built a character at a time with `+=`, a string is a rope, one node per + * character until something flattens it, and nothing in a read did. A + * workbook's 2.3 million shared characters held ~70 MB that way, against + * ~7 MB decoded flat. TextDecoder would be wrong twice over: the Encoding + * Standard maps "latin1" to windows-1252, which moves 0x80–0x9F, and + * "utf-16le" turns a lone surrogate into U+FFFD — which is what each half + * of a pair becomes when a CONTINUE boundary splits it. + */ +export function decodeChars( + buf: Uint8Array, + start: number, + count: number, + compressed: boolean, +): string { + const units = compressed ? buf.subarray(start, start + count) : utf16Units(buf, start, count) + if (units.length <= CHARS_PER_CALL) return String.fromCharCode(...units) + const parts: string[] = [] + for (let i = 0; i < units.length; i += CHARS_PER_CALL) { + parts.push(String.fromCharCode(...units.subarray(i, i + CHARS_PER_CALL))) + } + return parts.join("") +} + +function utf16Units(buf: Uint8Array, start: number, count: number): Uint16Array { + const units = new Uint16Array(count) + for (let i = 0, p = start; i < count; i++, p += 2) units[i] = buf[p] | (buf[p + 1] << 8) + return units +} + /** Decode an RK number (MS-XLS §2.5.166). */ export function decodeRk(rk: number): number { const fX100 = (rk & 1) !== 0 @@ -171,6 +260,12 @@ class BlockStream { this.pos += 4 return v } + /** `count` characters from the current block, which holds them all. */ + chars(count: number, compressed: boolean): string { + const s = decodeChars(this.cur(), this.pos, count, compressed) + this.pos += compressed ? count : count * 2 + return s + } skip(n: number): void { // skip may cross blocks (rich/phonetic trailers), no grbit byte let left = n @@ -224,6 +319,7 @@ function readSstString(s: BlockStream): string { const cRun = rich ? s.u16() : 0 const cbExt = phonetic ? s.u32() : 0 + // A block's worth of characters at a time — see decodeChars. let str = "" let read = 0 while (read < cch) { @@ -234,8 +330,17 @@ function readSstString(s: BlockStream): string { grbit = s.u8() compressed = (grbit & 0x01) === 0 } - str += String.fromCharCode(compressed ? s.u8() : s.u16()) - read++ + const fit = compressed ? s.remainingInBlock() : s.remainingInBlock() >> 1 + if (fit > 0) { + const take = Math.min(cch - read, fit) + str += s.chars(take, compressed) + read += take + } else { + // A two-byte character straddling the end of a block is malformed; + // it reads exactly as it always has. + str += String.fromCharCode(compressed ? s.u8() : s.u16()) + read++ + } } if (rich) s.skip(cRun * 4) if (phonetic) s.skip(cbExt) diff --git a/src/xls/reader.ts b/src/xls/reader.ts index feaeceda..634c04ac 100644 --- a/src/xls/reader.ts +++ b/src/xls/reader.ts @@ -11,7 +11,16 @@ import { MAX_COL_INDEX, MAX_ROW_INDEX, MAX_TOTAL_CELLS } from "../limits" import { readInputToUint8Array } from "../_input" import { readCfb } from "../xlsx/crypto/cfb" import { isBuiltinDateFormatId, isDateFormat, serialToDate } from "../_date" -import { decodeRk, parseRecords, parseSst, Reader, SID, type BiffRecord } from "./biff" +import { + decodeChars, + decodeRk, + parseSst, + Reader, + RecordCursor, + recordStarts, + SID, + type BiffRecord, +} from "./biff" import { codepageDecoder } from "./codepage" const ERROR_TEXT: Record = { @@ -106,8 +115,9 @@ function biff5Strings(decode: (bytes: Uint8Array) => string): BiffStrings { * of the globals pass because the strings that pass decodes — sheet names, * format codes — may in principle precede it. */ -function findCodepage(records: BiffRecord[]): number | undefined { - for (const rec of records) { +function findCodepage(stream: Uint8Array): number | undefined { + const cursor = new RecordCursor(stream) + for (let rec = cursor.next(); rec; rec = cursor.next()) { if (rec.id === SID.EOF) return undefined if (rec.id === SID.CODEPAGE) { if (rec.data.length < 2) throw new ParseError("Invalid XLS: CODEPAGE record is too short") @@ -118,7 +128,7 @@ function findCodepage(records: BiffRecord[]): number | undefined { } function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbook { - const records = parseRecords(stream) + const cursor = new RecordCursor(stream) // ── BIFF version gate ── // The first record is the workbook globals BOF; its first u16 is the BIFF @@ -126,7 +136,7 @@ function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbo // are Excel 5.0 and 95 — the same record layout except that strings are // code-page bytes and there is no shared-string table. Anything else is // rejected rather than misread. - const bof = records[0] + const bof = cursor.next() if (!bof || bof.id !== SID.BOF) { throw new ParseError("Invalid XLS: missing BOF record at start of Workbook stream") } @@ -145,28 +155,18 @@ function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbo // any string is read, so a bad page fails here rather than mid-sheet. const strings: BiffStrings = biffVersion === BIFF5 - ? biff5Strings( - codepageDecoder(findCodepage(records) ?? options?.codepage ?? DEFAULT_CODEPAGE), - ) + ? biff5Strings(codepageDecoder(findCodepage(stream) ?? options?.codepage ?? DEFAULT_CODEPAGE)) : BIFF8_STRINGS - const offsetToIndex = new Map() - for (let i = 0; i < records.length; i++) offsetToIndex.set(records[i].offset, i) - - // ── Globals substream (records[0] = BOF … first EOF) ── + // ── Globals substream (BOF … first EOF) ── let date1904 = options?.dateSystem === "1904" const xfFmtIds: number[] = [] const fmtCodes = new Map() const boundSheets: Array<{ name: string; pos: number }> = [] const sst: string[] = [] - let gi = 0 - for (; gi < records.length; gi++) { - const rec = records[gi] - if (rec.id === SID.EOF) { - gi++ - break - } + for (let rec: BiffRecord | undefined = bof; rec; rec = cursor.next()) { + if (rec.id === SID.EOF) break switch (rec.id) { case SID.DATEMODE: { if (!options?.dateSystem || options.dateSystem === "auto") { @@ -199,9 +199,9 @@ function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbo case SID.SST: { const blocks: Uint8Array[] = [rec.data] // Gather trailing CONTINUE records belonging to the SST. - for (let j = gi + 1; j < records.length; j++) { - if (records[j].id !== SID.CONTINUE) break - blocks.push(records[j].data) + for (let next = cursor.peek(); next?.id === SID.CONTINUE; next = cursor.peek()) { + blocks.push(next.data) + cursor.next() } // A loop, not `push(...parseSst(blocks))`: spreading an array as // arguments puts one stack slot per element, so a workbook with a @@ -233,17 +233,19 @@ function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbo const isDate = (ixfe: number): boolean => dateXf[ixfe] === true // ── Sheet substreams ── + const starts = recordStarts( + stream, + boundSheets.map((bs) => bs.pos), + ) const sheets: Sheet[] = [] for (const bs of boundSheets) { - const startIdx = offsetToIndex.get(bs.pos) - if (startIdx === undefined) { + if (!starts.has(bs.pos)) { sheets.push({ name: bs.name, rows: [] }) continue } sheets.push( parseSheet( - records, - startIdx, + new RecordCursor(stream, bs.pos), bs.name, sst, strings, @@ -258,8 +260,7 @@ function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbo } function parseSheet( - records: BiffRecord[], - startIdx: number, + cursor: RecordCursor, name: string, sst: string[], strings: BiffStrings, @@ -304,8 +305,8 @@ function parseSheet( setCell(row, col, isDate(ixfe) ? serialToDate(n, date1904) : n) } - for (let i = startIdx + 1; i < records.length; i++) { - const rec = records[i] + cursor.skip() // the sheet's BOF + for (let rec = cursor.next(); rec; rec = cursor.next()) { if (rec.id === SID.EOF) break const r = new Reader(rec.data) switch (rec.id) { @@ -374,7 +375,7 @@ function parseSheet( setCell(row, col, ERROR_TEXT[b[2]] ?? "#ERR!") // error else if (kind === 0) { // string: value is in the following STRING record - const next = records[i + 1] + const next = cursor.peek() if (next && next.id === SID.STRING) setCell(row, col, strings.long(new Reader(next.data))) } @@ -449,6 +450,15 @@ function readShortString(r: Reader): string { function readChars(r: Reader, cch: number): string { const grbit = r.u8() const compressed = (grbit & 0x01) === 0 + const bytes = compressed ? cch : cch * 2 + if (bytes <= r.remaining()) { + const s = decodeChars(r.buf, r.pos, cch, compressed) + r.skip(bytes) + return s + } + // A string running past its record — truncated, or a long formula result + // continued into a CONTINUE record, which is not followed here — reads + // as it always has, a character at a time. let s = "" for (let i = 0; i < cch; i++) s += String.fromCharCode(compressed ? r.u8() : r.u16()) return s diff --git a/test/xls.test.ts b/test/xls.test.ts index fb702df0..219d1c35 100644 --- a/test/xls.test.ts +++ b/test/xls.test.ts @@ -39,6 +39,7 @@ const rkInt = (v: number): number[] => u32(((v << 2) | 2) >>> 0) const SID = { FORMULA: 0x0006, EOF: 0x000a, + CONTINUE: 0x003c, DATEMODE: 0x0022, FORMAT: 0x041e, NUMBER: 0x0203, @@ -231,6 +232,113 @@ describe("XLS (BIFF8) reader", () => { }) }) +// ── Strings decoded a run at a time ────────────────────────────────── +// Text is decoded a record's worth of characters at a time, not one by +// one. Excel splits a long shared string wherever its record fills, and +// each CONTINUE that resumes one restarts with its own flag byte, so a +// string can change width mid-way and a surrogate pair can be cut in two. +// These pass on main as well: they pin what the faster path must keep. + +const latin = (s: string): number[] => [...s].map((c) => c.charCodeAt(0)) +function utf16(s: string): number[] { + const out: number[] = [] + for (let i = 0; i < s.length; i++) out.push(...u16(s.charCodeAt(i))) + return out +} +const generalXf = (): number[] => + record(SID.XF, [...u16(0), ...u16(0), ...Array.from({ length: 16 }, () => 0)]) + +/** One sheet after the given globals records, its BOUNDSHEET pointing at it. */ +function oneSheetXls(globals: number[][], sheet: number[][], sheetName = "Sheet1"): Uint8Array { + const makeGlobals = (sheetPos: number): Uint8Array => + concat([ + bof(0x0005), + ...globals, + record(SID.BOUNDSHEET, [...u32(sheetPos), 0, 0, ...shortStr(sheetName)]), + eof(), + ]) + const globalsStream = makeGlobals(makeGlobals(0).length) + return writeCfb([ + { name: "Workbook", data: concat([globalsStream, bof(0x0010), ...sheet, eof()]) }, + ]) +} + +describe("XLS (BIFF8) strings", () => { + it("joins a shared string's runs across CONTINUE records, whatever their width", async () => { + // 100 one-byte characters in the SST record, a CONTINUE record full of + // them (more than one String.fromCharCode call takes), then a switch + // to UTF-16 for the rest. The next string's surrogate pair straddles + // the following boundary. + const head = "A".repeat(100) + const middle = "B".repeat(8223) + const tail = "Ж".repeat(1677) + const emoji = "ab😀cd" + const xls = oneSheetXls( + [ + generalXf(), + record(SID.SST, [...u32(3), ...u32(3), ...u16(10_000), 0, ...latin(head)]), + record(SID.CONTINUE, [0, ...latin(middle)]), + record(SID.CONTINUE, [ + 1, + ...utf16(tail), + ...u16(emoji.length), + 1, + ...utf16(emoji.slice(0, 3)), + ]), + record(SID.CONTINUE, [1, ...utf16(emoji.slice(3)), ...u16(3), 0, ...latin("end")]), + ], + [0, 1, 2].map((i) => record(SID.LABELSST, [...u16(0), ...u16(i), ...u16(0), ...u32(i)])), + ) + + const wb = await readXls(xls) + + expect(wb.sheets[0].rows[0]).toEqual([head + middle + tail, emoji, "end"]) + }) + + it("reads inline text in either width, longer than one String.fromCharCode call takes", async () => { + const long = "x".repeat(8200) + const xls = oneSheetXls( + [generalXf()], + [ + record(SID.LABEL, [...u16(0), ...u16(0), ...u16(0), ...xlStr(long)]), + record(SID.LABEL, [...u16(0), ...u16(1), ...u16(0), ...u16(5), 1, ...utf16("Ωmega")]), + ], + ) + + const wb = await readXls(xls) + + expect(wb.sheets[0].rows[0]).toEqual([long, "Ωmega"]) + }) +}) + +describe("XLS (BIFF8) sheet offsets", () => { + it("reads a sheet whose BOUNDSHEET offset misses a record boundary as empty", async () => { + // The offset is the only link from a BOUNDSHEET to its sheet. Records + // are walked in place now rather than indexed by offset, and one that + // lands inside a record still means no rows, not records out of phase. + const makeGlobals = (sheetPos: number): Uint8Array => + concat([ + bof(0x0005), + generalXf(), + record(SID.BOUNDSHEET, [...u32(sheetPos), 0, 0, ...shortStr("Good")]), + record(SID.BOUNDSHEET, [...u32(sheetPos + 2), 0, 0, ...shortStr("Off")]), + eof(), + ]) + const globals = makeGlobals(makeGlobals(0).length) + const sheet = concat([ + bof(0x0010), + record(SID.LABEL, [...u16(0), ...u16(0), ...u16(0), ...xlStr("Hi")]), + eof(), + ]) + + const wb = await readXls(writeCfb([{ name: "Workbook", data: concat([globals, sheet]) }])) + + expect(wb.sheets.map((s) => s.name)).toEqual(["Good", "Off"]) + expect(wb.sheets[0].rows).toEqual([["Hi"]]) + expect(wb.sheets[1].rows).toEqual([]) + }) +}) + // ── Minimal BIFF5 .xls builder (test-only) ─────────────────────────── // Excel 5.0/95: the same records, but every string is bytes in the // workbook's code page with no per-string flag, cell text is written From b71602086a74de472a6c943ed33f82e867351d2e Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 21 Oct 2026 14:46:29 +0300 Subject: [PATCH 2/3] feat(xls): stream a sheet's rows with streamXlsRows --- README.md | 12 + docs/PARITY.md | 23 +- src/_stream-rows.ts | 71 +++++ src/index.ts | 1 + src/xls/reader.ts | 344 ++++++++++++++---------- src/xls/stream-reader.ts | 183 +++++++++++++ src/xlsx.ts | 1 + src/xlsx/stream-reader.ts | 72 +---- test/__snapshots__/exports.test.ts.snap | 2 + test/_xls-stream.ts | 40 +++ test/exports.test.ts | 2 +- test/real-files.test.ts | 24 ++ test/xls.test.ts | 172 ++++++++++++ 13 files changed, 727 insertions(+), 220 deletions(-) create mode 100644 src/_stream-rows.ts create mode 100644 src/xls/stream-reader.ts create mode 100644 test/_xls-stream.ts diff --git a/README.md b/README.md index 30c8c034..f09701f4 100644 --- a/README.md +++ b/README.md @@ -325,6 +325,7 @@ Process large files row-by-row without loading everything into memory: ```ts import { + streamXlsRows, streamXlsxRows, writeXlsxStream, writeXlsxStreamSheets, @@ -363,6 +364,13 @@ for await (const row of streamXlsxRows(buffer, { range: "B2:D1000" })) { // row.values[1..3] carry B/C/D } +// Legacy .xls streams the same way, with the same `sheet`, `range` and +// `maxRows`. The file is read whole — a CFB container needs random +// access — but the sheet's rows are never all held at once. +for await (const row of streamXlsRows(xlsBytes, { sheet: "Data" })) { + console.log(row.index, row.values) +} + // Stream write — the workbook is emitted as bytes while rows are pulled // from the source, so peak memory doesn't grow with the row count. function* rows() { @@ -720,6 +728,9 @@ Decodes the shared-string table (with CONTINUE spanning), RK / MULRK / number / boolean / error cells, labels, cached formula values, dates, and merged cells. Read-only. +For a sheet too large to hold as a workbook, `streamXlsRows` yields the +same values a row at a time — see [Streaming](#streaming). + Both legacy readers surface sheet names, cell values, and merges — and nothing else. Formulas arrive as their **cached value**, never as formula text; styles, column widths, row heights, sheet visibility, named ranges @@ -2031,6 +2042,7 @@ Zero dependencies. Pure TypeScript. The ZIP engine uses `CompressionStream`/`Dec | `openXlsx(input, options?)` | Open for round-trip (preserves unknown parts) | | `saveXlsx(workbook)` | Save round-trip workbook back to XLSX | | `streamXlsxRows(input, options?)` | AsyncGenerator yielding rows one at a time | +| `streamXlsRows(input, options?)` | The same for a legacy `.xls` sheet (BIFF5/BIFF8) | | `writeXlsxStream(rows, options)` | Constant-memory XLSX writing — returns a `ReadableStream` | | `toWriteOptions(workbook, opts?)` | Convert a read `Workbook` into `WriteOptions`; `onDrop` reports what it could not carry | | `XlsxStreamWriter` | Incremental XLSX writing (`addRow`/`addObject`); auto-splits past `maxRowsPerSheet` | diff --git a/docs/PARITY.md b/docs/PARITY.md index 6b21f59a..cc52aad6 100644 --- a/docs/PARITY.md +++ b/docs/PARITY.md @@ -607,16 +607,25 @@ Code that walks a sheet generically — `sheetToObjects`, `toHtml`, `row[i] ?? null` rather than assume a slot exists. That is what they all do; it is written down here because nothing said so. -The streaming readers _do_ agree: `streamXlsxRows` and `streamOdsRows` -both skip an entirely empty row and keep the true index on `StreamRow`, -so a gap in the indexes is the signal. +The streaming readers _do_ agree: `streamXlsxRows`, `streamXlsRows` and +`streamOdsRows` all skip an entirely empty row and keep the true index on +`StreamRow`, so a gap in the indexes is the signal. -They do **not** agree on how many sheets they walk. `streamXlsxRows` -yields one sheet — the first, unless `sheet` names another — +They do **not** agree on how many sheets they walk. `streamXlsxRows` and +`streamXlsRows` yield one sheet — the first, unless `sheet` names another — while `streamOdsRows` walks every sheet in the document and tags each row with `sheetIndex`. So the same loop over a three-sheet workbook gives you -one sheet of it as `.xlsx` and all three as `.ods`, silently. Pass -`sheet` when you mean one, and read `row.sheetIndex` when you mean all. +one sheet of it as `.xlsx` or `.xls` and all three as `.ods`, silently. +Pass `sheet` when you mean one, and read `row.sheetIndex` when you mean all. + +`streamXlsRows` reads the `.xls` itself whole, since a CFB container needs +random access, and streams the sheet out of it, so its memory follows the +file and its shared strings rather than the sheet. To stream, the cells +have to come in row order — the order BIFF lays its row blocks out in, +and the one Excel writes. +A sheet whose records go back a row is buffered and yielded in order +instead, holding its cells as `readXls` would, under the same +`maxTotalCells` bound. ### `streamXmlRows` gives you a row's own keys, not a rectangle diff --git a/src/_stream-rows.ts b/src/_stream-rows.ts new file mode 100644 index 00000000..c9202ada --- /dev/null +++ b/src/_stream-rows.ts @@ -0,0 +1,71 @@ +// ── Row filters shared by the streaming readers ────────────────────── +// `sheet`, `range` and `maxRows` mean the same in `streamXlsxRows` and +// `streamXlsRows`, so the two apply one implementation of each. + +import type { CellValue, StreamRow } from "./_types" +import { ParseError } from "./errors" +import { parseCellRef } from "./xlsx/worksheet" + +export interface RangeFilter { + startRow: number + endRow: number + startCol: number + endCol: number +} + +/** + * Parse a range reference like "A1:D10" into 0-based row/col bounds. + * Single-cell refs like "B2" are also accepted (start == end). + */ +export function parseRangeFilter(ref: string): RangeFilter { + const parts = ref.split(":") + if (parts.length === 0 || parts.length > 2) { + throw new ParseError(`Invalid range reference: "${ref}"`) + } + const start = parseCellRef(parts[0]!) + const end = parts.length > 1 ? parseCellRef(parts[1]!) : start + return { + startRow: Math.min(start.row, end.row), + endRow: Math.max(start.row, end.row), + startCol: Math.min(start.col, end.col), + endCol: Math.max(start.col, end.col), + } +} + +/** + * Apply the range filter to a freshly-yielded row. Returns the row to emit + * (with cells outside the column range nulled out) or `null` if the row + * itself falls outside the row range. + * + * Mirrors the batch reader (`parseWorksheet`): values stay aligned to + * their original column index, and cells outside the column window are + * masked to `null` rather than removed, so callers can still address + * `row.values[colIndex]` for columns inside the range. + */ +export function applyRangeFilter(row: StreamRow, range: RangeFilter): StreamRow | null { + if (row.index < range.startRow || row.index > range.endRow) return null + const len = Math.max(row.values.length, range.endCol + 1) + const out: CellValue[] = Array.from({ length: len }, () => null) + const upper = Math.min(row.values.length - 1, range.endCol) + for (let c = range.startCol; c <= upper; c++) { + out[c] = row.values[c] ?? null + } + return { index: row.index, values: out } +} + +/** The sheet a stream reads: the first, else the one `sheetSpec` names by 0-based index or by name. */ +export function resolveTargetSheet( + allSheets: T[], + sheetSpec?: number | string, +): T | null { + if (sheetSpec === undefined) { + // Default: first sheet + return allSheets[0] ?? null + } + + if (typeof sheetSpec === "number") { + return sheetSpec >= 0 && sheetSpec < allSheets.length ? allSheets[sheetSpec] : null + } + + return allSheets.find((s) => s.name === sheetSpec) ?? null +} diff --git a/src/index.ts b/src/index.ts index f0b9743d..b6bd6c0a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,7 @@ export type { ReadObjectsOptions, ReadObjectsResult, WriteObjectsTableOption } f export { readXlsx } from "./xlsx/reader" export { readXlsb } from "./xlsx/xlsb/reader" export { readXls } from "./xls/reader" +export { streamXlsRows } from "./xls/stream-reader" export { writeXlsx } from "./xlsx/writer" export { link } from "./xlsx/hyperlink" export { openXlsx, saveXlsx } from "./xlsx/roundtrip" diff --git a/src/xls/reader.ts b/src/xls/reader.ts index 634c04ac..9b3684d4 100644 --- a/src/xls/reader.ts +++ b/src/xls/reader.ts @@ -49,6 +49,19 @@ export async function readXls( input: Uint8Array | ArrayBuffer | ReadableStream, options?: ReadOptions, ): Promise { + const stream = await openWorkbookStream(input, options) + try { + return parseWorkbookRecords(stream, options) + } catch (err) { + throw asParseError(err) + } +} + +/** The container's Workbook (BIFF8) or Book (BIFF5) stream, read whole. */ +export async function openWorkbookStream( + input: Uint8Array | ArrayBuffer | ReadableStream, + options?: ReadOptions, +): Promise { const data = await readInputToUint8Array(input, options?.maxInputBytes) let streams: Map try { @@ -60,18 +73,20 @@ export async function readXls( } const stream = streams.get("Workbook") ?? streams.get("Book") if (!stream) throw new ParseError("Invalid XLS: missing Workbook stream") + return stream +} - // Record parsing reads many length-prefixed binary fields; a truncated or - // hostile file can make DataView accessors throw a raw RangeError. Wrap the - // whole pass so malformed input surfaces as the library's ParseError. - try { - return parseWorkbookRecords(stream, options) - } catch (err) { - if (err instanceof ParseError) throw err - throw new ParseError("Failed to parse XLS workbook (malformed or truncated)", undefined, { - cause: err, - }) - } +/** + * Record parsing reads many length-prefixed binary fields; a truncated or + * hostile file can make DataView accessors throw a raw RangeError. Each + * reader wraps its whole pass in this, so malformed input surfaces as the + * library's ParseError. + */ +export function asParseError(err: unknown): ParseError { + if (err instanceof ParseError) return err + return new ParseError("Failed to parse XLS workbook (malformed or truncated)", undefined, { + cause: err, + }) } /** @@ -81,7 +96,7 @@ export async function readXls( * length prefix is a u16 on cell text and a u8 on names and format codes * in both. */ -interface BiffStrings { +export interface BiffStrings { long(r: Reader): string short(r: Reader): string } @@ -127,7 +142,44 @@ function findCodepage(stream: Uint8Array): number | undefined { return undefined } +/** What the globals substream settles for every sheet: how to read text and dates, and where the sheets are. */ +export interface XlsGlobals { + strings: BiffStrings + sst: string[] + isDate: (ixfe: number) => boolean + date1904: boolean + boundSheets: Array<{ name: string; pos: number }> +} + function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbook { + const globals = readGlobals(stream, options) + + // ── Sheet substreams ── + const starts = recordStarts( + stream, + globals.boundSheets.map((bs) => bs.pos), + ) + const sheets: Sheet[] = [] + for (const bs of globals.boundSheets) { + if (!starts.has(bs.pos)) { + sheets.push({ name: bs.name, rows: [] }) + continue + } + sheets.push( + parseSheet( + new RecordCursor(stream, bs.pos), + bs.name, + globals, + options?.maxTotalCells ?? MAX_TOTAL_CELLS, + ), + ) + } + + return { sheets } +} + +/** The globals substream (BOF … first EOF), read once for any reader. */ +export function readGlobals(stream: Uint8Array, options?: ReadOptions): XlsGlobals { const cursor = new RecordCursor(stream) // ── BIFF version gate ── @@ -232,40 +284,13 @@ function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbo const isDate = (ixfe: number): boolean => dateXf[ixfe] === true - // ── Sheet substreams ── - const starts = recordStarts( - stream, - boundSheets.map((bs) => bs.pos), - ) - const sheets: Sheet[] = [] - for (const bs of boundSheets) { - if (!starts.has(bs.pos)) { - sheets.push({ name: bs.name, rows: [] }) - continue - } - sheets.push( - parseSheet( - new RecordCursor(stream, bs.pos), - bs.name, - sst, - strings, - isDate, - date1904, - options?.maxTotalCells ?? MAX_TOTAL_CELLS, - ), - ) - } - - return { sheets } + return { strings, sst, isDate, date1904, boundSheets } } function parseSheet( cursor: RecordCursor, name: string, - sst: string[], - strings: BiffStrings, - isDate: (ixfe: number) => boolean, - date1904: boolean, + globals: XlsGlobals, cellLimit: number, ): Sheet { const rows: CellValue[][] = [] @@ -278,16 +303,7 @@ function parseSheet( // not. See #363. let widestCol = 0 const setCell = (row: number, col: number, value: CellValue): void => { - if (row < 0 || row > MAX_ROW_INDEX) { - throw new ParseError( - `Cell row ${row} is outside the supported sheet bounds (max ${MAX_ROW_INDEX + 1})`, - ) - } - if (col < 0 || col > MAX_COL_INDEX) { - throw new ParseError( - `Cell column ${col} is outside the supported sheet bounds (max ${MAX_COL_INDEX + 1})`, - ) - } + assertInSheetBounds(row, col) if (col >= widestCol) widestCol = col + 1 const boundingBox = Math.max(rows.length, row + 1) * widestCol if (boundingBox > cellLimit) { @@ -298,107 +314,24 @@ function parseSheet( } let r = rows[row] if (!r) r = rows[row] = [] - while (r.length < col) r.push(null) - r[col] = value - } - const numeric = (row: number, col: number, ixfe: number, n: number): void => { - setCell(row, col, isDate(ixfe) ? serialToDate(n, date1904) : n) + placeCell(r, col, value) } cursor.skip() // the sheet's BOF for (let rec = cursor.next(); rec; rec = cursor.next()) { if (rec.id === SID.EOF) break + if (rec.id !== SID.MERGECELLS) { + decodeCells(rec, cursor, globals, setCell) + continue + } const r = new Reader(rec.data) - switch (rec.id) { - case SID.LABELSST: { - const row = r.u16(), - col = r.u16() - r.u16() // ixfe - setCell(row, col, sst[r.u32()] ?? "") - break - } - case SID.RK: { - const row = r.u16(), - col = r.u16(), - ixfe = r.u16() - numeric(row, col, ixfe, decodeRk(r.u32())) - break - } - case SID.NUMBER: { - const row = r.u16(), - col = r.u16(), - ixfe = r.u16() - numeric(row, col, ixfe, r.f64()) - break - } - case SID.MULRK: { - const row = r.u16() - const colFirst = r.u16() - const count = (rec.data.length - 6) / 6 - for (let k = 0; k < count; k++) { - const ixfe = r.u16() - numeric(row, colFirst + k, ixfe, decodeRk(r.u32())) - } - break - } - case SID.BOOLERR: { - const row = r.u16(), - col = r.u16() - r.u16() // ixfe - const val = r.u8() - const isError = r.u8() === 1 - setCell(row, col, isError ? (ERROR_TEXT[val] ?? "#ERR!") : val !== 0) - break - } - case SID.LABEL: - case SID.RSTRING: { - // Same head; RSTRING (BIFF5 rich text) trails formatting runs the - // value does not carry. - const row = r.u16(), - col = r.u16() - r.u16() // ixfe - const text = strings.long(r) - if (rec.id === SID.RSTRING) skipBiff5Runs(r) - setCell(row, col, text) - break - } - case SID.FORMULA: { - const row = r.u16(), - col = r.u16(), - ixfe = r.u16() - const b = rec.data.subarray(r.pos, r.pos + 8) - if (b[6] === 0xff && b[7] === 0xff) { - const kind = b[0] - if (kind === 1) - setCell(row, col, b[2] !== 0) // boolean - else if (kind === 2) - setCell(row, col, ERROR_TEXT[b[2]] ?? "#ERR!") // error - else if (kind === 0) { - // string: value is in the following STRING record - const next = cursor.peek() - if (next && next.id === SID.STRING) - setCell(row, col, strings.long(new Reader(next.data))) - } - // kind === 3 → blank/empty - } else { - const num = new DataView(b.buffer, b.byteOffset, 8).getFloat64(0, true) - numeric(row, col, ixfe, num) - } - break - } - case SID.MERGECELLS: { - const cmcs = r.u16() - for (let k = 0; k < cmcs; k++) { - const rwFirst = r.u16(), - rwLast = r.u16(), - colFirst = r.u16(), - colLast = r.u16() - merges.push({ startRow: rwFirst, endRow: rwLast, startCol: colFirst, endCol: colLast }) - } - break - } - default: - break + const cmcs = r.u16() + for (let k = 0; k < cmcs; k++) { + const rwFirst = r.u16(), + rwLast = r.u16(), + colFirst = r.u16(), + colLast = r.u16() + merges.push({ startRow: rwFirst, endRow: rwLast, startCol: colFirst, endCol: colLast }) } } @@ -416,6 +349,125 @@ function parseSheet( return sheet } +/** BIFF rows and columns are u16s; the model is narrower, and a cell past it is refused, not dropped. */ +export function assertInSheetBounds(row: number, col: number): void { + if (row < 0 || row > MAX_ROW_INDEX) { + throw new ParseError( + `Cell row ${row} is outside the supported sheet bounds (max ${MAX_ROW_INDEX + 1})`, + ) + } + if (col < 0 || col > MAX_COL_INDEX) { + throw new ParseError( + `Cell column ${col} is outside the supported sheet bounds (max ${MAX_COL_INDEX + 1})`, + ) + } +} + +/** Write `value` at `col`, padding the row with nulls up to it. */ +export function placeCell(row: CellValue[], col: number, value: CellValue): void { + while (row.length < col) row.push(null) + row[col] = value +} + +/** + * The cells one record carries, each handed to `set` in the record's own + * order — every reader decodes values here, so they cannot drift apart. + * Records that carry no value (BLANK, MERGECELLS, formatting) pass through. + * A string formula's cached value is in the STRING record after it, which + * `cursor` still points at. + */ +export function decodeCells( + rec: BiffRecord, + cursor: RecordCursor, + globals: XlsGlobals, + set: (row: number, col: number, value: CellValue) => void, +): void { + const { sst, strings, isDate, date1904 } = globals + const numeric = (row: number, col: number, ixfe: number, n: number): void => { + set(row, col, isDate(ixfe) ? serialToDate(n, date1904) : n) + } + const r = new Reader(rec.data) + switch (rec.id) { + case SID.LABELSST: { + const row = r.u16(), + col = r.u16() + r.u16() // ixfe + set(row, col, sst[r.u32()] ?? "") + break + } + case SID.RK: { + const row = r.u16(), + col = r.u16(), + ixfe = r.u16() + numeric(row, col, ixfe, decodeRk(r.u32())) + break + } + case SID.NUMBER: { + const row = r.u16(), + col = r.u16(), + ixfe = r.u16() + numeric(row, col, ixfe, r.f64()) + break + } + case SID.MULRK: { + const row = r.u16() + const colFirst = r.u16() + const count = (rec.data.length - 6) / 6 + for (let k = 0; k < count; k++) { + const ixfe = r.u16() + numeric(row, colFirst + k, ixfe, decodeRk(r.u32())) + } + break + } + case SID.BOOLERR: { + const row = r.u16(), + col = r.u16() + r.u16() // ixfe + const val = r.u8() + const isError = r.u8() === 1 + set(row, col, isError ? (ERROR_TEXT[val] ?? "#ERR!") : val !== 0) + break + } + case SID.LABEL: + case SID.RSTRING: { + // Same head; RSTRING (BIFF5 rich text) trails formatting runs the + // value does not carry. + const row = r.u16(), + col = r.u16() + r.u16() // ixfe + const text = strings.long(r) + if (rec.id === SID.RSTRING) skipBiff5Runs(r) + set(row, col, text) + break + } + case SID.FORMULA: { + const row = r.u16(), + col = r.u16(), + ixfe = r.u16() + const b = rec.data.subarray(r.pos, r.pos + 8) + if (b[6] === 0xff && b[7] === 0xff) { + const kind = b[0] + if (kind === 1) + set(row, col, b[2] !== 0) // boolean + else if (kind === 2) + set(row, col, ERROR_TEXT[b[2]] ?? "#ERR!") // error + else if (kind === 0) { + // string: value is in the following STRING record + const next = cursor.peek() + if (next && next.id === SID.STRING) set(row, col, strings.long(new Reader(next.data))) + } + // kind === 3 → blank/empty + } else { + const num = new DataView(b.buffer, b.byteOffset, 8).getFloat64(0, true) + numeric(row, col, ixfe, num) + } + break + } + default: + break + } +} + /** * The BIFF5 RSTRING trailer: a u8 run count, then two bytes per run (first * character, font). Narrower than BIFF8's FormatRun (u16 + u16), which is diff --git a/src/xls/stream-reader.ts b/src/xls/stream-reader.ts new file mode 100644 index 00000000..791afb27 --- /dev/null +++ b/src/xls/stream-reader.ts @@ -0,0 +1,183 @@ +// ── Streaming XLS Reader ───────────────────────────────────────────── +// Yields one sheet of a BIFF5/BIFF8 .xls a row at a time. The container +// is read whole — a CFB file's directory and allocation tables can sit +// anywhere in it — but the sheet is never materialized: values are +// decoded from the records as the cursor reaches them, by the same code +// `readXls` uses. + +import type { CellValue, ReadOptions, StreamRow } from "../_types" +import { ParseError } from "../errors" +import { MAX_TOTAL_CELLS } from "../limits" +import { applyRangeFilter, parseRangeFilter, resolveTargetSheet } from "../_stream-rows" +import { RecordCursor, recordStarts, SID } from "./biff" +import { + asParseError, + assertInSheetBounds, + decodeCells, + openWorkbookStream, + placeCell, + readGlobals, + type XlsGlobals, +} from "./reader" + +/** + * Stream one sheet of a BIFF5 or BIFF8 .xls workbook, a row at a time. + * + * The first sheet, or the one `sheet` names by 0-based index or by name; + * a sheet that is not there yields nothing. `range` and `maxRows` mean + * what they do for `streamXlsxRows`, and reading stops as soon as no + * more rows can be emitted. As in the other stream readers, an empty row + * is skipped and `index` is the row's own — a gap in the indexes is the + * signal — and `values` run to the row's last cell, nulls in between. + * + * The input is read whole, because a CFB container needs random access; + * the rows are not, so memory follows the file and its shared strings + * rather than the sheet. Streaming needs the cells in row order — the + * order BIFF lays its row blocks out in, and the one Excel writes. A sheet + * whose records go back a row is buffered instead and yielded in order: + * its cells are held the way `readXls` holds them, under the same + * `maxTotalCells` bound. + */ +export async function* streamXlsRows( + input: Uint8Array | ArrayBuffer | ReadableStream, + options?: ReadOptions & { sheet?: number | string }, +): AsyncGenerator { + const stream = await openWorkbookStream(input, options) + let rows: Iterable + try { + const globals = readGlobals(stream, options) + const target = resolveTargetSheet(globals.boundSheets, options?.sheet) + // An offset that misses a record boundary is an empty sheet to readXls too. + if (!target || !recordStarts(stream, [target.pos]).has(target.pos)) return + rows = inRowOrder(stream, target.pos) + ? streamedRows(stream, target.pos, globals) + : bufferedRows(stream, target.pos, globals, options?.maxTotalCells ?? MAX_TOTAL_CELLS) + } catch (err) { + throw asParseError(err) + } + + const range = options?.range ? parseRangeFilter(options.range) : undefined + const maxRows = typeof options?.maxRows === "number" && options.maxRows > 0 ? options.maxRows : 0 + let emitted = 0 + for (const row of rows) { + // Rows ascend, so past the end of the range nothing more can match. + if (range && row.index > range.endRow) return + const filtered = range ? applyRangeFilter(row, range) : row + if (!filtered) continue + yield filtered + if (maxRows > 0 && ++emitted >= maxRows) return + } +} + +/** The records that carry a cell value — the ones whose rows have to ascend for a sheet to stream. */ +const VALUE_RECORDS: ReadonlySet = new Set([ + SID.LABELSST, + SID.RK, + SID.NUMBER, + SID.MULRK, + SID.BOOLERR, + SID.LABEL, + SID.RSTRING, + SID.FORMULA, +]) + +/** + * Whether no value record goes back a row. Read from the record headers + * alone, before anything is decoded, so a sheet that needs buffering is + * known before its first row is yielded. + */ +function inRowOrder(stream: Uint8Array, pos: number): boolean { + const view = new DataView(stream.buffer, stream.byteOffset, stream.byteLength) + const cursor = new RecordCursor(stream, pos) + cursor.skip() // the sheet's BOF + let last = 0 + for (let at = cursor.skip(); at >= 0; at = cursor.skip()) { + const id = view.getUint16(at, true) + if (id === SID.EOF) break + // A value record too short to name its row is malformed, which decoding it reports. + if (!VALUE_RECORDS.has(id) || view.getUint16(at + 2, true) < 2 || at + 6 > stream.length) { + continue + } + const row = view.getUint16(at + 4, true) + if (row < last) return false + last = row + } + return true +} + +/** The sheet's rows as the records reach them: a row is complete when a later row's cell arrives. */ +function* streamedRows( + stream: Uint8Array, + pos: number, + globals: XlsGlobals, +): Generator { + try { + const cursor = new RecordCursor(stream, pos) + cursor.skip() // the sheet's BOF + let index = -1 + let values: CellValue[] = [] + // One record's cells share a row, so a record completes at most one row. + const ready: StreamRow[] = [] + const set = (row: number, col: number, value: CellValue): void => { + assertInSheetBounds(row, col) + if (row !== index) { + if (index >= 0) ready.push({ index, values }) + index = row + values = [] + } + placeCell(values, col, value) + } + for (let rec = cursor.next(); rec; rec = cursor.next()) { + if (rec.id === SID.EOF) break + decodeCells(rec, cursor, globals, set) + for (const row of ready) yield row + ready.length = 0 + } + if (index >= 0) yield { index, values } + } catch (err) { + throw asParseError(err) + } +} + +/** + * A sheet whose records go back a row, held whole and then yielded in + * order, so no row is yielded twice or split in two. Its cells are kept + * per row rather than as a rectangle, but bounded as `readXls` bounds the + * rectangle it would allocate. + */ +function* bufferedRows( + stream: Uint8Array, + pos: number, + globals: XlsGlobals, + cellLimit: number, +): Generator { + const byRow = new Map() + try { + const cursor = new RecordCursor(stream, pos) + cursor.skip() // the sheet's BOF + let rowCount = 0 + let widestCol = 0 + const set = (row: number, col: number, value: CellValue): void => { + assertInSheetBounds(row, col) + if (row >= rowCount) rowCount = row + 1 + if (col >= widestCol) widestCol = col + 1 + if (rowCount * widestCol > cellLimit) { + throw new ParseError( + `Sheet spans ${rowCount * widestCol} cells, over the ${cellLimit} limit. ` + + "Raise `maxTotalCells` if the sheet really is this large.", + ) + } + let values = byRow.get(row) + if (!values) byRow.set(row, (values = [])) + placeCell(values, col, value) + } + for (let rec = cursor.next(); rec; rec = cursor.next()) { + if (rec.id === SID.EOF) break + decodeCells(rec, cursor, globals, set) + } + } catch (err) { + throw asParseError(err) + } + const ordered = [...byRow.entries()].sort((a, b) => a[0] - b[0]) + for (const [index, values] of ordered) yield { index, values } +} diff --git a/src/xlsx.ts b/src/xlsx.ts index 4290778a..20117b56 100644 --- a/src/xlsx.ts +++ b/src/xlsx.ts @@ -5,6 +5,7 @@ export { readXlsx } from "./xlsx/reader" export { writeXlsx } from "./xlsx/writer" export { readXlsb } from "./xlsx/xlsb/reader" export { readXls } from "./xls/reader" +export { streamXlsRows } from "./xls/stream-reader" export { readXlsxObjects, writeXlsxObjects } from "./xlsx/objects" export type { XlsxObjectsReadOptions, diff --git a/src/xlsx/stream-reader.ts b/src/xlsx/stream-reader.ts index 8f89e03a..e7bbcfff 100644 --- a/src/xlsx/stream-reader.ts +++ b/src/xlsx/stream-reader.ts @@ -21,6 +21,12 @@ import { parseSharedStrings } from "./shared-strings" import { parseStyles, isDateStyle } from "./styles" import { parseCellRef, parseIsoCellDate } from "./worksheet" import { serialToDate } from "../_date" +import { + applyRangeFilter, + parseRangeFilter, + resolveTargetSheet, + type RangeFilter, +} from "../_stream-rows" // ── Types ──────────────────────────────────────────────────────────── @@ -29,34 +35,6 @@ import { serialToDate } from "../_date" // `import type { StreamRow } from "hucre/xlsx"` keeps working. export type { StreamRow } from "../_types" -// ── Range filter ──────────────────────────────────────────────────── - -interface RangeFilter { - startRow: number - endRow: number - startCol: number - endCol: number -} - -/** - * Parse a range reference like "A1:D10" into 0-based row/col bounds. - * Single-cell refs like "B2" are also accepted (start == end). - */ -function parseRangeFilter(ref: string): RangeFilter { - const parts = ref.split(":") - if (parts.length === 0 || parts.length > 2) { - throw new ParseError(`Invalid range reference: "${ref}"`) - } - const start = parseCellRef(parts[0]!) - const end = parts.length > 1 ? parseCellRef(parts[1]!) : start - return { - startRow: Math.min(start.row, end.row), - endRow: Math.max(start.row, end.row), - startCol: Math.min(start.col, end.col), - endCol: Math.max(start.col, end.col), - } -} - // ── OOXML Relationship Types ───────────────────────────────────────── // Use lenient matching (matchesRelType) so both Transitional and Strict // OOXML namespace URIs are accepted, matching the batch reader. @@ -144,21 +122,6 @@ function parseWorkbookXml( return { sheets, dateSystem } } -// ── Resolve target sheet ──────────────────────────────────────────── - -function resolveTargetSheet(allSheets: SheetInfo[], sheetSpec?: number | string): SheetInfo | null { - if (sheetSpec === undefined) { - // Default: first sheet - return allSheets[0] ?? null - } - - if (typeof sheetSpec === "number") { - return sheetSpec >= 0 && sheetSpec < allSheets.length ? allSheets[sheetSpec] : null - } - - return allSheets.find((s) => s.name === sheetSpec) ?? null -} - // ── Worksheet SAX handlers (shared between sync and streaming paths) ─ interface RowSaxState { @@ -362,29 +325,6 @@ function handleCloseTag( return null } -// ── Filter application ───────────────────────────────────────────── - -/** - * Apply the range filter to a freshly-yielded row. Returns the row to emit - * (with cells outside the column range nulled out) or `null` if the row - * itself falls outside the row range. - * - * Mirrors the batch reader (`parseWorksheet`): values stay aligned to - * their original column index, and cells outside the column window are - * masked to `null` rather than removed, so callers can still address - * `row.values[colIndex]` for columns inside the range. - */ -function applyRangeFilter(row: StreamRow, range: RangeFilter): StreamRow | null { - if (row.index < range.startRow || row.index > range.endRow) return null - const len = Math.max(row.values.length, range.endCol + 1) - const out: CellValue[] = Array.from({ length: len }, () => null) - const upper = Math.min(row.values.length - 1, range.endCol) - for (let c = range.startCol; c <= upper; c++) { - out[c] = row.values[c] ?? null - } - return { index: row.index, values: out } -} - // ── Streaming row parser via SAX (async — ReadableStream) ────────── async function* parseWorksheetRowsStreaming( diff --git a/test/__snapshots__/exports.test.ts.snap b/test/__snapshots__/exports.test.ts.snap index 2a404f87..8076864a 100644 --- a/test/__snapshots__/exports.test.ts.snap +++ b/test/__snapshots__/exports.test.ts.snap @@ -104,6 +104,7 @@ exports[`export surface snapshot > hucre/xlsx 1`] = ` "readXlsxObjects", "resolveThemeColor", "saveXlsx", + "streamXlsRows", "streamXlsxRows", "toRange", "toRanges", @@ -246,6 +247,7 @@ exports[`export surface snapshot > root 1`] = ` "streamCsvRows", "streamNdjsonRows", "streamOdsRows", + "streamXlsRows", "streamXlsxRows", "streamXmlRows", "stripBom", diff --git a/test/_xls-stream.ts b/test/_xls-stream.ts new file mode 100644 index 00000000..aad310f6 --- /dev/null +++ b/test/_xls-stream.ts @@ -0,0 +1,40 @@ +import { expect } from "vitest" +import type { CellValue, ReadOptions, StreamRow } from "../src/_types" +import { readXls } from "../src/xls/reader" +import { streamXlsRows } from "../src/xls/stream-reader" + +/** Every row a stream yields, collected. */ +export async function streamed( + xls: Uint8Array, + options?: ReadOptions & { sheet?: number | string }, +): Promise { + const out: StreamRow[] = [] + for await (const row of streamXlsRows(xls, options)) out.push(row) + return out +} + +/** + * A dense sheet as a stream yields it: the empty rows left out, each row + * cut after its last value. A value is never null, so trailing nulls are + * only the rectangle's padding. + */ +export function asStreamed(rows: CellValue[][]): StreamRow[] { + const out: StreamRow[] = [] + rows.forEach((row, index) => { + let end = row.length + while (end > 0 && row[end - 1] === null) end-- + if (end > 0) out.push({ index, values: row.slice(0, end) }) + }) + return out +} + +/** Each sheet, streamed, holds exactly what readXls reads for it. */ +export async function expectStreamMatchesReadXls( + xls: Uint8Array, + options?: ReadOptions, +): Promise { + const workbook = await readXls(xls, options) + for (const [sheet, { rows }] of workbook.sheets.entries()) { + expect(await streamed(xls, { ...options, sheet })).toEqual(asStreamed(rows)) + } +} diff --git a/test/exports.test.ts b/test/exports.test.ts index 182f8d2b..6cda9f2e 100644 --- a/test/exports.test.ts +++ b/test/exports.test.ts @@ -149,7 +149,7 @@ describe("hucre/xlsx", () => { }) it("exports the other formats it can read", () => { - expectExports(xlsx, ["readXlsb", "readXls"], "hucre/xlsx") + expectExports(xlsx, ["readXlsb", "readXls", "streamXlsRows"], "hucre/xlsx") }) it("exports the streaming API", () => { diff --git a/test/real-files.test.ts b/test/real-files.test.ts index 7c41d00e..3bc27807 100644 --- a/test/real-files.test.ts +++ b/test/real-files.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest" import { readFileSync } from "node:fs" import { read, readXls, readXlsb, readXlsx, streamXlsxRows } from "../src/index" import type { Cell, CellStyle, ReadOptions, Sheet, Workbook } from "../src/_types" +import { expectStreamMatchesReadXls } from "./_xls-stream" // ═══════════════════════════════════════════════════════════════════════ // #464 — files hucre did not write. @@ -372,6 +373,29 @@ describe("workbooks written by Excel, not by this test suite", () => { // leaking into the model. PARITY.md already says XLS and XLSB carry // less than XLSX — styles, dimensions, properties — so the comparison // is limited to the values, which all three do claim to carry. + // streamXlsRows decodes with readXls's code but walks the records its + // own way — a row at a time, ending each row when the next one starts — + // so files from two producers check it agrees with readXls on every + // value, every row and every sheet. + describe("streamXlsRows streams what readXls reads", () => { + const files = [ + ...FIXTURES.filter((f) => f.reader === readXls).map((f) => f.file), + ...[ + "sheetjs-dates.xls", + "sheetjs-shared-strings.xls", + "sheetjs-sparse.xls", + "sheetjs-unicode.xls", + "sheetjs-values.xls", + "sheetjs-wide.xls", + ].map((file) => `third-party/${file}`), + ] + for (const file of files) { + it(file, async () => { + await expectStreamMatchesReadXls(bytes(file)) + }) + } + }) + describe("one sheet, three containers", () => { const rowsOf = async (file: string, reader: typeof readXlsx): Promise => (await modelOf(file, reader)).sheets[0]?.rows ?? [] diff --git a/test/xls.test.ts b/test/xls.test.ts index 219d1c35..598f9434 100644 --- a/test/xls.test.ts +++ b/test/xls.test.ts @@ -1,7 +1,10 @@ import { describe, expect, it } from "vitest" import { writeCfb } from "../src/xlsx/crypto/cfb" import { readXls } from "../src/xls/reader" +import { streamXlsRows } from "../src/xls/stream-reader" import { read } from "../src/defter" +import { ParseError } from "../src/errors" +import { expectStreamMatchesReadXls, streamed } from "./_xls-stream" // ── Minimal BIFF8 .xls builder (test-only) ─────────────────────────── @@ -52,6 +55,7 @@ const SID = { XF: 0x00e0, BOUNDSHEET: 0x0085, MERGECELLS: 0x00e5, + BLANK: 0x0201, BOF: 0x0809, } @@ -535,3 +539,171 @@ describe("XLS (BIFF5) reader", () => { expect(wb.sheets[0].rows[0][0]).toBe("Видаткова накладна") }) }) + +// ── streamXlsRows ──────────────────────────────────────────────────── +// The stream decodes values with readXls's own code, so most of what +// follows is about shape: which sheet, which rows, in what order, and +// when it stops reading. + +const label = (row: number, col: number, text: string): number[] => + record(SID.LABEL, [...u16(row), ...u16(col), ...u16(0), ...xlStr(text)]) + +/** Two sheets, each holding its own name in A1. */ +function twoSheetXls(): Uint8Array { + const sheets = ["First", "Second"].map((name) => concat([bof(0x0010), label(0, 0, name), eof()])) + const makeGlobals = (firstPos: number): Uint8Array => + concat([ + bof(0x0005), + generalXf(), + record(SID.BOUNDSHEET, [...u32(firstPos), 0, 0, ...shortStr("First")]), + record(SID.BOUNDSHEET, [...u32(firstPos + sheets[0].length), 0, 0, ...shortStr("Second")]), + eof(), + ]) + const globals = makeGlobals(makeGlobals(0).length) + return writeCfb([{ name: "Workbook", data: concat([globals, ...sheets]) }]) +} + +/** Rows 0–4, the cell in column c of row r reading `${r}${c}`; rows 1 and 2 four columns wide. */ +const gridRecords = (): number[][] => [ + label(0, 0, "00"), + ...[0, 1, 2, 3].map((c) => label(1, c, `1${c}`)), + ...[0, 1, 2, 3].map((c) => label(2, c, `2${c}`)), + label(3, 0, "30"), + label(4, 0, "40"), +] + +/** A LABEL record cut off after its row — malformed wherever it sits. */ +const truncatedLabel = (row: number): number[] => record(SID.LABEL, u16(row)) + +describe("streamXlsRows", () => { + describe("streams what readXls reads", () => { + it("BIFF8: SST labels, RK, MULRK, numbers, bools, errors and dates", async () => { + await expectStreamMatchesReadXls(buildXls()) + }) + + it("BIFF8: a FORMAT record that redefines a built-in id", async () => { + await expectStreamMatchesReadXls( + buildXls({ dateFmtId: 50, fmtCodes: [[50, "000000000000"]], value: 81227827687 }), + ) + }) + + it("BIFF8: the 1904 date system, when asked for", async () => { + await expectStreamMatchesReadXls(buildXls(), { dateSystem: "1904" }) + }) + + it("BIFF5: code-page text, RSTRING, and a string formula's STRING record", async () => { + await expectStreamMatchesReadXls(buildXls5({ codepage: 1251 })) + await expectStreamMatchesReadXls(buildXls5(), { codepage: 1251 }) + }) + }) + + it("streams the first sheet, or the one named by index or by name", async () => { + const xls = twoSheetXls() + + expect(await streamed(xls)).toEqual([{ index: 0, values: ["First"] }]) + expect(await streamed(xls, { sheet: 1 })).toEqual([{ index: 0, values: ["Second"] }]) + expect(await streamed(xls, { sheet: "Second" })).toEqual([{ index: 0, values: ["Second"] }]) + }) + + it("yields nothing for a sheet that is not there", async () => { + const xls = twoSheetXls() + + expect(await streamed(xls, { sheet: 2 })).toEqual([]) + expect(await streamed(xls, { sheet: "Third" })).toEqual([]) + }) + + it("skips an empty row and keeps each row's own index", async () => { + // Row 1 carries only a BLANK — formatting, no value — and row 2 nothing at all. + const xls = oneSheetXls( + [generalXf()], + [label(0, 0, "a"), record(SID.BLANK, [...u16(1), ...u16(0), ...u16(0)]), label(3, 2, "d")], + ) + + expect(await streamed(xls)).toEqual([ + { index: 0, values: ["a"] }, + { index: 3, values: [null, null, "d"] }, + ]) + }) + + it("applies range as streamXlsxRows does: rows outside skipped, columns outside masked", async () => { + const xls = oneSheetXls([generalXf()], gridRecords()) + + expect(await streamed(xls, { range: "B2:C3" })).toEqual([ + { index: 1, values: [null, "11", "12", null] }, + { index: 2, values: [null, "21", "22", null] }, + ]) + }) + + it("stops reading once range or maxRows can emit nothing more", async () => { + // Everything after row 4 is damaged: reading that far is an error, so + // a read that succeeds is one that stopped before it. + const xls = oneSheetXls([generalXf()], [...gridRecords(), truncatedLabel(5)]) + + await expect(streamed(xls)).rejects.toThrow(ParseError) + expect((await streamed(xls, { range: "A1:D3" })).map((row) => row.index)).toEqual([0, 1, 2]) + expect((await streamed(xls, { maxRows: 2 })).map((row) => row.index)).toEqual([0, 1]) + }) + + it("buffers a sheet whose cells go back a row, and yields each row once, in order", async () => { + // Streaming such a sheet would have to yield row 0 twice, or split it. + const xls = oneSheetXls( + [generalXf()], + [label(2, 0, "c"), label(0, 0, "a"), label(1, 0, "b"), label(0, 1, "a2")], + ) + + expect(await streamed(xls)).toEqual([ + { index: 0, values: ["a", "a2"] }, + { index: 1, values: ["b"] }, + { index: 2, values: ["c"] }, + ]) + await expectStreamMatchesReadXls(xls) + }) + + it("bounds a buffered sheet by maxTotalCells as readXls bounds it, and a streamed one not at all", async () => { + const backwards = oneSheetXls([generalXf()], [label(2, 1, "c"), label(0, 0, "a")]) + const forwards = oneSheetXls([generalXf()], [label(0, 0, "a"), label(2, 1, "c")]) + + await expect(readXls(backwards, { maxTotalCells: 4 })).rejects.toThrow(/maxTotalCells/) + await expect(streamed(backwards, { maxTotalCells: 4 })).rejects.toThrow(/maxTotalCells/) + // Nothing is held while streaming, so there is nothing for the bound to bound. + expect(await streamed(forwards, { maxTotalCells: 4 })).toHaveLength(2) + }) + + it("reports damage as a ParseError, before the first row and mid-sheet alike", async () => { + await expect(streamed(new Uint8Array([1, 2, 3, 4]))).rejects.toThrow(ParseError) + const xls = oneSheetXls([generalXf()], [label(0, 0, "a"), truncatedLabel(1)]) + await expect(streamed(xls)).rejects.toThrow(ParseError) + }) + + it("refuses a column past the sheet's bounds, as readXls does", async () => { + const xls = oneSheetXls([generalXf()], [label(0, 20_000, "far")]) + + await expect(readXls(xls)).rejects.toThrow(/outside the supported sheet bounds/) + await expect(streamed(xls)).rejects.toThrow(/outside the supported sheet bounds/) + }) + + it("takes a ReadableStream, as the other readers do, bounded by maxInputBytes", async () => { + const bytes = buildXls() + const input = (): ReadableStream => + new ReadableStream({ + start(controller) { + controller.enqueue(bytes) + controller.close() + }, + }) + const rows = [] + for await (const row of streamXlsRows(input())) rows.push(row) + + expect(rows).toEqual(await streamed(bytes)) + await expect(readXls(input(), { maxInputBytes: 64 })).rejects.toThrow(ParseError) + await expect(streamXlsRows(input(), { maxInputBytes: 64 }).next()).rejects.toThrow(ParseError) + }) + + it("can be abandoned after its first row", async () => { + const rows = streamXlsRows(buildXls()) + + expect((await rows.next()).value).toEqual({ index: 0, values: ["Name", "Score"] }) + expect(await rows.return(undefined)).toEqual({ done: true, value: undefined }) + expect(await rows.next()).toEqual({ done: true, value: undefined }) + }) +}) From dab39a4a4f1e916f0f09019061207b9d8063afa5 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 21 Oct 2026 15:00:01 +0300 Subject: [PATCH 3/3] docs(xls): say which sheets streamXlsRows buffers --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f09701f4..90837b27 100644 --- a/README.md +++ b/README.md @@ -366,7 +366,9 @@ for await (const row of streamXlsxRows(buffer, { range: "B2:D1000" })) { // Legacy .xls streams the same way, with the same `sheet`, `range` and // `maxRows`. The file is read whole — a CFB container needs random -// access — but the sheet's rows are never all held at once. +// access — and a sheet whose cells come in row order is never held +// whole; one whose records go back a row is buffered, under +// `maxTotalCells`, and yielded in order. for await (const row of streamXlsRows(xlsBytes, { sheet: "Data" })) { console.log(row.index, row.values) }