diff --git a/README.md b/README.md index 30c8c034..90837b27 100644 --- a/README.md +++ b/README.md @@ -325,6 +325,7 @@ Process large files row-by-row without loading everything into memory: ```ts import { + streamXlsRows, streamXlsxRows, writeXlsxStream, writeXlsxStreamSheets, @@ -363,6 +364,15 @@ for await (const row of streamXlsxRows(buffer, { range: "B2:D1000" })) { // row.values[1..3] carry B/C/D } +// Legacy .xls streams the same way, with the same `sheet`, `range` and +// `maxRows`. The file is read whole — a CFB container needs random +// access — and a sheet whose cells come in row order is never held +// whole; one whose records go back a row is buffered, under +// `maxTotalCells`, and yielded in order. +for await (const row of streamXlsRows(xlsBytes, { sheet: "Data" })) { + console.log(row.index, row.values) +} + // Stream write — the workbook is emitted as bytes while rows are pulled // from the source, so peak memory doesn't grow with the row count. function* rows() { @@ -720,6 +730,9 @@ Decodes the shared-string table (with CONTINUE spanning), RK / MULRK / number / boolean / error cells, labels, cached formula values, dates, and merged cells. Read-only. +For a sheet too large to hold as a workbook, `streamXlsRows` yields the +same values a row at a time — see [Streaming](#streaming). + Both legacy readers surface sheet names, cell values, and merges — and nothing else. Formulas arrive as their **cached value**, never as formula text; styles, column widths, row heights, sheet visibility, named ranges @@ -2031,6 +2044,7 @@ Zero dependencies. Pure TypeScript. The ZIP engine uses `CompressionStream`/`Dec | `openXlsx(input, options?)` | Open for round-trip (preserves unknown parts) | | `saveXlsx(workbook)` | Save round-trip workbook back to XLSX | | `streamXlsxRows(input, options?)` | AsyncGenerator yielding rows one at a time | +| `streamXlsRows(input, options?)` | The same for a legacy `.xls` sheet (BIFF5/BIFF8) | | `writeXlsxStream(rows, options)` | Constant-memory XLSX writing — returns a `ReadableStream` | | `toWriteOptions(workbook, opts?)` | Convert a read `Workbook` into `WriteOptions`; `onDrop` reports what it could not carry | | `XlsxStreamWriter` | Incremental XLSX writing (`addRow`/`addObject`); auto-splits past `maxRowsPerSheet` | diff --git a/docs/PARITY.md b/docs/PARITY.md index 76c7b578..06e00ceb 100644 --- a/docs/PARITY.md +++ b/docs/PARITY.md @@ -680,16 +680,25 @@ Code that walks a sheet generically — `sheetToObjects`, `toHtml`, `row[i] ?? null` rather than assume a slot exists. That is what they all do; it is written down here because nothing said so. -The streaming readers _do_ agree: `streamXlsxRows` and `streamOdsRows` -both skip an entirely empty row and keep the true index on `StreamRow`, -so a gap in the indexes is the signal. +The streaming readers _do_ agree: `streamXlsxRows`, `streamXlsRows` and +`streamOdsRows` all skip an entirely empty row and keep the true index on +`StreamRow`, so a gap in the indexes is the signal. -They do **not** agree on how many sheets they walk. `streamXlsxRows` -yields one sheet — the first, unless `sheet` names another — +They do **not** agree on how many sheets they walk. `streamXlsxRows` and +`streamXlsRows` yield one sheet — the first, unless `sheet` names another — while `streamOdsRows` walks every sheet in the document and tags each row with `sheetIndex`. So the same loop over a three-sheet workbook gives you -one sheet of it as `.xlsx` and all three as `.ods`, silently. Pass -`sheet` when you mean one, and read `row.sheetIndex` when you mean all. +one sheet of it as `.xlsx` or `.xls` and all three as `.ods`, silently. +Pass `sheet` when you mean one, and read `row.sheetIndex` when you mean all. + +`streamXlsRows` reads the `.xls` itself whole, since a CFB container needs +random access, and streams the sheet out of it, so its memory follows the +file and its shared strings rather than the sheet. To stream, the cells +have to come in row order — the order BIFF lays its row blocks out in, +and the one Excel writes. +A sheet whose records go back a row is buffered and yielded in order +instead, holding its cells as `readXls` would, under the same +`maxTotalCells` bound. ### `streamXmlRows` gives you a row's own keys, not a rectangle diff --git a/src/_stream-rows.ts b/src/_stream-rows.ts new file mode 100644 index 00000000..c9202ada --- /dev/null +++ b/src/_stream-rows.ts @@ -0,0 +1,71 @@ +// ── Row filters shared by the streaming readers ────────────────────── +// `sheet`, `range` and `maxRows` mean the same in `streamXlsxRows` and +// `streamXlsRows`, so the two apply one implementation of each. + +import type { CellValue, StreamRow } from "./_types" +import { ParseError } from "./errors" +import { parseCellRef } from "./xlsx/worksheet" + +export interface RangeFilter { + startRow: number + endRow: number + startCol: number + endCol: number +} + +/** + * Parse a range reference like "A1:D10" into 0-based row/col bounds. + * Single-cell refs like "B2" are also accepted (start == end). + */ +export function parseRangeFilter(ref: string): RangeFilter { + const parts = ref.split(":") + if (parts.length === 0 || parts.length > 2) { + throw new ParseError(`Invalid range reference: "${ref}"`) + } + const start = parseCellRef(parts[0]!) + const end = parts.length > 1 ? parseCellRef(parts[1]!) : start + return { + startRow: Math.min(start.row, end.row), + endRow: Math.max(start.row, end.row), + startCol: Math.min(start.col, end.col), + endCol: Math.max(start.col, end.col), + } +} + +/** + * Apply the range filter to a freshly-yielded row. Returns the row to emit + * (with cells outside the column range nulled out) or `null` if the row + * itself falls outside the row range. + * + * Mirrors the batch reader (`parseWorksheet`): values stay aligned to + * their original column index, and cells outside the column window are + * masked to `null` rather than removed, so callers can still address + * `row.values[colIndex]` for columns inside the range. + */ +export function applyRangeFilter(row: StreamRow, range: RangeFilter): StreamRow | null { + if (row.index < range.startRow || row.index > range.endRow) return null + const len = Math.max(row.values.length, range.endCol + 1) + const out: CellValue[] = Array.from({ length: len }, () => null) + const upper = Math.min(row.values.length - 1, range.endCol) + for (let c = range.startCol; c <= upper; c++) { + out[c] = row.values[c] ?? null + } + return { index: row.index, values: out } +} + +/** The sheet a stream reads: the first, else the one `sheetSpec` names by 0-based index or by name. */ +export function resolveTargetSheet( + allSheets: T[], + sheetSpec?: number | string, +): T | null { + if (sheetSpec === undefined) { + // Default: first sheet + return allSheets[0] ?? null + } + + if (typeof sheetSpec === "number") { + return sheetSpec >= 0 && sheetSpec < allSheets.length ? allSheets[sheetSpec] : null + } + + return allSheets.find((s) => s.name === sheetSpec) ?? null +} diff --git a/src/index.ts b/src/index.ts index f0b9743d..b6bd6c0a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,7 @@ export type { ReadObjectsOptions, ReadObjectsResult, WriteObjectsTableOption } f export { readXlsx } from "./xlsx/reader" export { readXlsb } from "./xlsx/xlsb/reader" export { readXls } from "./xls/reader" +export { streamXlsRows } from "./xls/stream-reader" export { writeXlsx } from "./xlsx/writer" export { link } from "./xlsx/hyperlink" export { openXlsx, saveXlsx } from "./xlsx/roundtrip" diff --git a/src/xls/reader.ts b/src/xls/reader.ts index 634c04ac..9b3684d4 100644 --- a/src/xls/reader.ts +++ b/src/xls/reader.ts @@ -49,6 +49,19 @@ export async function readXls( input: Uint8Array | ArrayBuffer | ReadableStream, options?: ReadOptions, ): Promise { + const stream = await openWorkbookStream(input, options) + try { + return parseWorkbookRecords(stream, options) + } catch (err) { + throw asParseError(err) + } +} + +/** The container's Workbook (BIFF8) or Book (BIFF5) stream, read whole. */ +export async function openWorkbookStream( + input: Uint8Array | ArrayBuffer | ReadableStream, + options?: ReadOptions, +): Promise { const data = await readInputToUint8Array(input, options?.maxInputBytes) let streams: Map try { @@ -60,18 +73,20 @@ export async function readXls( } const stream = streams.get("Workbook") ?? streams.get("Book") if (!stream) throw new ParseError("Invalid XLS: missing Workbook stream") + return stream +} - // Record parsing reads many length-prefixed binary fields; a truncated or - // hostile file can make DataView accessors throw a raw RangeError. Wrap the - // whole pass so malformed input surfaces as the library's ParseError. - try { - return parseWorkbookRecords(stream, options) - } catch (err) { - if (err instanceof ParseError) throw err - throw new ParseError("Failed to parse XLS workbook (malformed or truncated)", undefined, { - cause: err, - }) - } +/** + * Record parsing reads many length-prefixed binary fields; a truncated or + * hostile file can make DataView accessors throw a raw RangeError. Each + * reader wraps its whole pass in this, so malformed input surfaces as the + * library's ParseError. + */ +export function asParseError(err: unknown): ParseError { + if (err instanceof ParseError) return err + return new ParseError("Failed to parse XLS workbook (malformed or truncated)", undefined, { + cause: err, + }) } /** @@ -81,7 +96,7 @@ export async function readXls( * length prefix is a u16 on cell text and a u8 on names and format codes * in both. */ -interface BiffStrings { +export interface BiffStrings { long(r: Reader): string short(r: Reader): string } @@ -127,7 +142,44 @@ function findCodepage(stream: Uint8Array): number | undefined { return undefined } +/** What the globals substream settles for every sheet: how to read text and dates, and where the sheets are. */ +export interface XlsGlobals { + strings: BiffStrings + sst: string[] + isDate: (ixfe: number) => boolean + date1904: boolean + boundSheets: Array<{ name: string; pos: number }> +} + function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbook { + const globals = readGlobals(stream, options) + + // ── Sheet substreams ── + const starts = recordStarts( + stream, + globals.boundSheets.map((bs) => bs.pos), + ) + const sheets: Sheet[] = [] + for (const bs of globals.boundSheets) { + if (!starts.has(bs.pos)) { + sheets.push({ name: bs.name, rows: [] }) + continue + } + sheets.push( + parseSheet( + new RecordCursor(stream, bs.pos), + bs.name, + globals, + options?.maxTotalCells ?? MAX_TOTAL_CELLS, + ), + ) + } + + return { sheets } +} + +/** The globals substream (BOF … first EOF), read once for any reader. */ +export function readGlobals(stream: Uint8Array, options?: ReadOptions): XlsGlobals { const cursor = new RecordCursor(stream) // ── BIFF version gate ── @@ -232,40 +284,13 @@ function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbo const isDate = (ixfe: number): boolean => dateXf[ixfe] === true - // ── Sheet substreams ── - const starts = recordStarts( - stream, - boundSheets.map((bs) => bs.pos), - ) - const sheets: Sheet[] = [] - for (const bs of boundSheets) { - if (!starts.has(bs.pos)) { - sheets.push({ name: bs.name, rows: [] }) - continue - } - sheets.push( - parseSheet( - new RecordCursor(stream, bs.pos), - bs.name, - sst, - strings, - isDate, - date1904, - options?.maxTotalCells ?? MAX_TOTAL_CELLS, - ), - ) - } - - return { sheets } + return { strings, sst, isDate, date1904, boundSheets } } function parseSheet( cursor: RecordCursor, name: string, - sst: string[], - strings: BiffStrings, - isDate: (ixfe: number) => boolean, - date1904: boolean, + globals: XlsGlobals, cellLimit: number, ): Sheet { const rows: CellValue[][] = [] @@ -278,16 +303,7 @@ function parseSheet( // not. See #363. let widestCol = 0 const setCell = (row: number, col: number, value: CellValue): void => { - if (row < 0 || row > MAX_ROW_INDEX) { - throw new ParseError( - `Cell row ${row} is outside the supported sheet bounds (max ${MAX_ROW_INDEX + 1})`, - ) - } - if (col < 0 || col > MAX_COL_INDEX) { - throw new ParseError( - `Cell column ${col} is outside the supported sheet bounds (max ${MAX_COL_INDEX + 1})`, - ) - } + assertInSheetBounds(row, col) if (col >= widestCol) widestCol = col + 1 const boundingBox = Math.max(rows.length, row + 1) * widestCol if (boundingBox > cellLimit) { @@ -298,107 +314,24 @@ function parseSheet( } let r = rows[row] if (!r) r = rows[row] = [] - while (r.length < col) r.push(null) - r[col] = value - } - const numeric = (row: number, col: number, ixfe: number, n: number): void => { - setCell(row, col, isDate(ixfe) ? serialToDate(n, date1904) : n) + placeCell(r, col, value) } cursor.skip() // the sheet's BOF for (let rec = cursor.next(); rec; rec = cursor.next()) { if (rec.id === SID.EOF) break + if (rec.id !== SID.MERGECELLS) { + decodeCells(rec, cursor, globals, setCell) + continue + } const r = new Reader(rec.data) - switch (rec.id) { - case SID.LABELSST: { - const row = r.u16(), - col = r.u16() - r.u16() // ixfe - setCell(row, col, sst[r.u32()] ?? "") - break - } - case SID.RK: { - const row = r.u16(), - col = r.u16(), - ixfe = r.u16() - numeric(row, col, ixfe, decodeRk(r.u32())) - break - } - case SID.NUMBER: { - const row = r.u16(), - col = r.u16(), - ixfe = r.u16() - numeric(row, col, ixfe, r.f64()) - break - } - case SID.MULRK: { - const row = r.u16() - const colFirst = r.u16() - const count = (rec.data.length - 6) / 6 - for (let k = 0; k < count; k++) { - const ixfe = r.u16() - numeric(row, colFirst + k, ixfe, decodeRk(r.u32())) - } - break - } - case SID.BOOLERR: { - const row = r.u16(), - col = r.u16() - r.u16() // ixfe - const val = r.u8() - const isError = r.u8() === 1 - setCell(row, col, isError ? (ERROR_TEXT[val] ?? "#ERR!") : val !== 0) - break - } - case SID.LABEL: - case SID.RSTRING: { - // Same head; RSTRING (BIFF5 rich text) trails formatting runs the - // value does not carry. - const row = r.u16(), - col = r.u16() - r.u16() // ixfe - const text = strings.long(r) - if (rec.id === SID.RSTRING) skipBiff5Runs(r) - setCell(row, col, text) - break - } - case SID.FORMULA: { - const row = r.u16(), - col = r.u16(), - ixfe = r.u16() - const b = rec.data.subarray(r.pos, r.pos + 8) - if (b[6] === 0xff && b[7] === 0xff) { - const kind = b[0] - if (kind === 1) - setCell(row, col, b[2] !== 0) // boolean - else if (kind === 2) - setCell(row, col, ERROR_TEXT[b[2]] ?? "#ERR!") // error - else if (kind === 0) { - // string: value is in the following STRING record - const next = cursor.peek() - if (next && next.id === SID.STRING) - setCell(row, col, strings.long(new Reader(next.data))) - } - // kind === 3 → blank/empty - } else { - const num = new DataView(b.buffer, b.byteOffset, 8).getFloat64(0, true) - numeric(row, col, ixfe, num) - } - break - } - case SID.MERGECELLS: { - const cmcs = r.u16() - for (let k = 0; k < cmcs; k++) { - const rwFirst = r.u16(), - rwLast = r.u16(), - colFirst = r.u16(), - colLast = r.u16() - merges.push({ startRow: rwFirst, endRow: rwLast, startCol: colFirst, endCol: colLast }) - } - break - } - default: - break + const cmcs = r.u16() + for (let k = 0; k < cmcs; k++) { + const rwFirst = r.u16(), + rwLast = r.u16(), + colFirst = r.u16(), + colLast = r.u16() + merges.push({ startRow: rwFirst, endRow: rwLast, startCol: colFirst, endCol: colLast }) } } @@ -416,6 +349,125 @@ function parseSheet( return sheet } +/** BIFF rows and columns are u16s; the model is narrower, and a cell past it is refused, not dropped. */ +export function assertInSheetBounds(row: number, col: number): void { + if (row < 0 || row > MAX_ROW_INDEX) { + throw new ParseError( + `Cell row ${row} is outside the supported sheet bounds (max ${MAX_ROW_INDEX + 1})`, + ) + } + if (col < 0 || col > MAX_COL_INDEX) { + throw new ParseError( + `Cell column ${col} is outside the supported sheet bounds (max ${MAX_COL_INDEX + 1})`, + ) + } +} + +/** Write `value` at `col`, padding the row with nulls up to it. */ +export function placeCell(row: CellValue[], col: number, value: CellValue): void { + while (row.length < col) row.push(null) + row[col] = value +} + +/** + * The cells one record carries, each handed to `set` in the record's own + * order — every reader decodes values here, so they cannot drift apart. + * Records that carry no value (BLANK, MERGECELLS, formatting) pass through. + * A string formula's cached value is in the STRING record after it, which + * `cursor` still points at. + */ +export function decodeCells( + rec: BiffRecord, + cursor: RecordCursor, + globals: XlsGlobals, + set: (row: number, col: number, value: CellValue) => void, +): void { + const { sst, strings, isDate, date1904 } = globals + const numeric = (row: number, col: number, ixfe: number, n: number): void => { + set(row, col, isDate(ixfe) ? serialToDate(n, date1904) : n) + } + const r = new Reader(rec.data) + switch (rec.id) { + case SID.LABELSST: { + const row = r.u16(), + col = r.u16() + r.u16() // ixfe + set(row, col, sst[r.u32()] ?? "") + break + } + case SID.RK: { + const row = r.u16(), + col = r.u16(), + ixfe = r.u16() + numeric(row, col, ixfe, decodeRk(r.u32())) + break + } + case SID.NUMBER: { + const row = r.u16(), + col = r.u16(), + ixfe = r.u16() + numeric(row, col, ixfe, r.f64()) + break + } + case SID.MULRK: { + const row = r.u16() + const colFirst = r.u16() + const count = (rec.data.length - 6) / 6 + for (let k = 0; k < count; k++) { + const ixfe = r.u16() + numeric(row, colFirst + k, ixfe, decodeRk(r.u32())) + } + break + } + case SID.BOOLERR: { + const row = r.u16(), + col = r.u16() + r.u16() // ixfe + const val = r.u8() + const isError = r.u8() === 1 + set(row, col, isError ? (ERROR_TEXT[val] ?? "#ERR!") : val !== 0) + break + } + case SID.LABEL: + case SID.RSTRING: { + // Same head; RSTRING (BIFF5 rich text) trails formatting runs the + // value does not carry. + const row = r.u16(), + col = r.u16() + r.u16() // ixfe + const text = strings.long(r) + if (rec.id === SID.RSTRING) skipBiff5Runs(r) + set(row, col, text) + break + } + case SID.FORMULA: { + const row = r.u16(), + col = r.u16(), + ixfe = r.u16() + const b = rec.data.subarray(r.pos, r.pos + 8) + if (b[6] === 0xff && b[7] === 0xff) { + const kind = b[0] + if (kind === 1) + set(row, col, b[2] !== 0) // boolean + else if (kind === 2) + set(row, col, ERROR_TEXT[b[2]] ?? "#ERR!") // error + else if (kind === 0) { + // string: value is in the following STRING record + const next = cursor.peek() + if (next && next.id === SID.STRING) set(row, col, strings.long(new Reader(next.data))) + } + // kind === 3 → blank/empty + } else { + const num = new DataView(b.buffer, b.byteOffset, 8).getFloat64(0, true) + numeric(row, col, ixfe, num) + } + break + } + default: + break + } +} + /** * The BIFF5 RSTRING trailer: a u8 run count, then two bytes per run (first * character, font). Narrower than BIFF8's FormatRun (u16 + u16), which is diff --git a/src/xls/stream-reader.ts b/src/xls/stream-reader.ts new file mode 100644 index 00000000..791afb27 --- /dev/null +++ b/src/xls/stream-reader.ts @@ -0,0 +1,183 @@ +// ── Streaming XLS Reader ───────────────────────────────────────────── +// Yields one sheet of a BIFF5/BIFF8 .xls a row at a time. The container +// is read whole — a CFB file's directory and allocation tables can sit +// anywhere in it — but the sheet is never materialized: values are +// decoded from the records as the cursor reaches them, by the same code +// `readXls` uses. + +import type { CellValue, ReadOptions, StreamRow } from "../_types" +import { ParseError } from "../errors" +import { MAX_TOTAL_CELLS } from "../limits" +import { applyRangeFilter, parseRangeFilter, resolveTargetSheet } from "../_stream-rows" +import { RecordCursor, recordStarts, SID } from "./biff" +import { + asParseError, + assertInSheetBounds, + decodeCells, + openWorkbookStream, + placeCell, + readGlobals, + type XlsGlobals, +} from "./reader" + +/** + * Stream one sheet of a BIFF5 or BIFF8 .xls workbook, a row at a time. + * + * The first sheet, or the one `sheet` names by 0-based index or by name; + * a sheet that is not there yields nothing. `range` and `maxRows` mean + * what they do for `streamXlsxRows`, and reading stops as soon as no + * more rows can be emitted. As in the other stream readers, an empty row + * is skipped and `index` is the row's own — a gap in the indexes is the + * signal — and `values` run to the row's last cell, nulls in between. + * + * The input is read whole, because a CFB container needs random access; + * the rows are not, so memory follows the file and its shared strings + * rather than the sheet. Streaming needs the cells in row order — the + * order BIFF lays its row blocks out in, and the one Excel writes. A sheet + * whose records go back a row is buffered instead and yielded in order: + * its cells are held the way `readXls` holds them, under the same + * `maxTotalCells` bound. + */ +export async function* streamXlsRows( + input: Uint8Array | ArrayBuffer | ReadableStream, + options?: ReadOptions & { sheet?: number | string }, +): AsyncGenerator { + const stream = await openWorkbookStream(input, options) + let rows: Iterable + try { + const globals = readGlobals(stream, options) + const target = resolveTargetSheet(globals.boundSheets, options?.sheet) + // An offset that misses a record boundary is an empty sheet to readXls too. + if (!target || !recordStarts(stream, [target.pos]).has(target.pos)) return + rows = inRowOrder(stream, target.pos) + ? streamedRows(stream, target.pos, globals) + : bufferedRows(stream, target.pos, globals, options?.maxTotalCells ?? MAX_TOTAL_CELLS) + } catch (err) { + throw asParseError(err) + } + + const range = options?.range ? parseRangeFilter(options.range) : undefined + const maxRows = typeof options?.maxRows === "number" && options.maxRows > 0 ? options.maxRows : 0 + let emitted = 0 + for (const row of rows) { + // Rows ascend, so past the end of the range nothing more can match. + if (range && row.index > range.endRow) return + const filtered = range ? applyRangeFilter(row, range) : row + if (!filtered) continue + yield filtered + if (maxRows > 0 && ++emitted >= maxRows) return + } +} + +/** The records that carry a cell value — the ones whose rows have to ascend for a sheet to stream. */ +const VALUE_RECORDS: ReadonlySet = new Set([ + SID.LABELSST, + SID.RK, + SID.NUMBER, + SID.MULRK, + SID.BOOLERR, + SID.LABEL, + SID.RSTRING, + SID.FORMULA, +]) + +/** + * Whether no value record goes back a row. Read from the record headers + * alone, before anything is decoded, so a sheet that needs buffering is + * known before its first row is yielded. + */ +function inRowOrder(stream: Uint8Array, pos: number): boolean { + const view = new DataView(stream.buffer, stream.byteOffset, stream.byteLength) + const cursor = new RecordCursor(stream, pos) + cursor.skip() // the sheet's BOF + let last = 0 + for (let at = cursor.skip(); at >= 0; at = cursor.skip()) { + const id = view.getUint16(at, true) + if (id === SID.EOF) break + // A value record too short to name its row is malformed, which decoding it reports. + if (!VALUE_RECORDS.has(id) || view.getUint16(at + 2, true) < 2 || at + 6 > stream.length) { + continue + } + const row = view.getUint16(at + 4, true) + if (row < last) return false + last = row + } + return true +} + +/** The sheet's rows as the records reach them: a row is complete when a later row's cell arrives. */ +function* streamedRows( + stream: Uint8Array, + pos: number, + globals: XlsGlobals, +): Generator { + try { + const cursor = new RecordCursor(stream, pos) + cursor.skip() // the sheet's BOF + let index = -1 + let values: CellValue[] = [] + // One record's cells share a row, so a record completes at most one row. + const ready: StreamRow[] = [] + const set = (row: number, col: number, value: CellValue): void => { + assertInSheetBounds(row, col) + if (row !== index) { + if (index >= 0) ready.push({ index, values }) + index = row + values = [] + } + placeCell(values, col, value) + } + for (let rec = cursor.next(); rec; rec = cursor.next()) { + if (rec.id === SID.EOF) break + decodeCells(rec, cursor, globals, set) + for (const row of ready) yield row + ready.length = 0 + } + if (index >= 0) yield { index, values } + } catch (err) { + throw asParseError(err) + } +} + +/** + * A sheet whose records go back a row, held whole and then yielded in + * order, so no row is yielded twice or split in two. Its cells are kept + * per row rather than as a rectangle, but bounded as `readXls` bounds the + * rectangle it would allocate. + */ +function* bufferedRows( + stream: Uint8Array, + pos: number, + globals: XlsGlobals, + cellLimit: number, +): Generator { + const byRow = new Map() + try { + const cursor = new RecordCursor(stream, pos) + cursor.skip() // the sheet's BOF + let rowCount = 0 + let widestCol = 0 + const set = (row: number, col: number, value: CellValue): void => { + assertInSheetBounds(row, col) + if (row >= rowCount) rowCount = row + 1 + if (col >= widestCol) widestCol = col + 1 + if (rowCount * widestCol > cellLimit) { + throw new ParseError( + `Sheet spans ${rowCount * widestCol} cells, over the ${cellLimit} limit. ` + + "Raise `maxTotalCells` if the sheet really is this large.", + ) + } + let values = byRow.get(row) + if (!values) byRow.set(row, (values = [])) + placeCell(values, col, value) + } + for (let rec = cursor.next(); rec; rec = cursor.next()) { + if (rec.id === SID.EOF) break + decodeCells(rec, cursor, globals, set) + } + } catch (err) { + throw asParseError(err) + } + const ordered = [...byRow.entries()].sort((a, b) => a[0] - b[0]) + for (const [index, values] of ordered) yield { index, values } +} diff --git a/src/xlsx.ts b/src/xlsx.ts index 4290778a..20117b56 100644 --- a/src/xlsx.ts +++ b/src/xlsx.ts @@ -5,6 +5,7 @@ export { readXlsx } from "./xlsx/reader" export { writeXlsx } from "./xlsx/writer" export { readXlsb } from "./xlsx/xlsb/reader" export { readXls } from "./xls/reader" +export { streamXlsRows } from "./xls/stream-reader" export { readXlsxObjects, writeXlsxObjects } from "./xlsx/objects" export type { XlsxObjectsReadOptions, diff --git a/src/xlsx/stream-reader.ts b/src/xlsx/stream-reader.ts index 8f89e03a..e7bbcfff 100644 --- a/src/xlsx/stream-reader.ts +++ b/src/xlsx/stream-reader.ts @@ -21,6 +21,12 @@ import { parseSharedStrings } from "./shared-strings" import { parseStyles, isDateStyle } from "./styles" import { parseCellRef, parseIsoCellDate } from "./worksheet" import { serialToDate } from "../_date" +import { + applyRangeFilter, + parseRangeFilter, + resolveTargetSheet, + type RangeFilter, +} from "../_stream-rows" // ── Types ──────────────────────────────────────────────────────────── @@ -29,34 +35,6 @@ import { serialToDate } from "../_date" // `import type { StreamRow } from "hucre/xlsx"` keeps working. export type { StreamRow } from "../_types" -// ── Range filter ──────────────────────────────────────────────────── - -interface RangeFilter { - startRow: number - endRow: number - startCol: number - endCol: number -} - -/** - * Parse a range reference like "A1:D10" into 0-based row/col bounds. - * Single-cell refs like "B2" are also accepted (start == end). - */ -function parseRangeFilter(ref: string): RangeFilter { - const parts = ref.split(":") - if (parts.length === 0 || parts.length > 2) { - throw new ParseError(`Invalid range reference: "${ref}"`) - } - const start = parseCellRef(parts[0]!) - const end = parts.length > 1 ? parseCellRef(parts[1]!) : start - return { - startRow: Math.min(start.row, end.row), - endRow: Math.max(start.row, end.row), - startCol: Math.min(start.col, end.col), - endCol: Math.max(start.col, end.col), - } -} - // ── OOXML Relationship Types ───────────────────────────────────────── // Use lenient matching (matchesRelType) so both Transitional and Strict // OOXML namespace URIs are accepted, matching the batch reader. @@ -144,21 +122,6 @@ function parseWorkbookXml( return { sheets, dateSystem } } -// ── Resolve target sheet ──────────────────────────────────────────── - -function resolveTargetSheet(allSheets: SheetInfo[], sheetSpec?: number | string): SheetInfo | null { - if (sheetSpec === undefined) { - // Default: first sheet - return allSheets[0] ?? null - } - - if (typeof sheetSpec === "number") { - return sheetSpec >= 0 && sheetSpec < allSheets.length ? allSheets[sheetSpec] : null - } - - return allSheets.find((s) => s.name === sheetSpec) ?? null -} - // ── Worksheet SAX handlers (shared between sync and streaming paths) ─ interface RowSaxState { @@ -362,29 +325,6 @@ function handleCloseTag( return null } -// ── Filter application ───────────────────────────────────────────── - -/** - * Apply the range filter to a freshly-yielded row. Returns the row to emit - * (with cells outside the column range nulled out) or `null` if the row - * itself falls outside the row range. - * - * Mirrors the batch reader (`parseWorksheet`): values stay aligned to - * their original column index, and cells outside the column window are - * masked to `null` rather than removed, so callers can still address - * `row.values[colIndex]` for columns inside the range. - */ -function applyRangeFilter(row: StreamRow, range: RangeFilter): StreamRow | null { - if (row.index < range.startRow || row.index > range.endRow) return null - const len = Math.max(row.values.length, range.endCol + 1) - const out: CellValue[] = Array.from({ length: len }, () => null) - const upper = Math.min(row.values.length - 1, range.endCol) - for (let c = range.startCol; c <= upper; c++) { - out[c] = row.values[c] ?? null - } - return { index: row.index, values: out } -} - // ── Streaming row parser via SAX (async — ReadableStream) ────────── async function* parseWorksheetRowsStreaming( diff --git a/test/__snapshots__/exports.test.ts.snap b/test/__snapshots__/exports.test.ts.snap index 2a404f87..8076864a 100644 --- a/test/__snapshots__/exports.test.ts.snap +++ b/test/__snapshots__/exports.test.ts.snap @@ -104,6 +104,7 @@ exports[`export surface snapshot > hucre/xlsx 1`] = ` "readXlsxObjects", "resolveThemeColor", "saveXlsx", + "streamXlsRows", "streamXlsxRows", "toRange", "toRanges", @@ -246,6 +247,7 @@ exports[`export surface snapshot > root 1`] = ` "streamCsvRows", "streamNdjsonRows", "streamOdsRows", + "streamXlsRows", "streamXlsxRows", "streamXmlRows", "stripBom", diff --git a/test/_xls-stream.ts b/test/_xls-stream.ts new file mode 100644 index 00000000..aad310f6 --- /dev/null +++ b/test/_xls-stream.ts @@ -0,0 +1,40 @@ +import { expect } from "vitest" +import type { CellValue, ReadOptions, StreamRow } from "../src/_types" +import { readXls } from "../src/xls/reader" +import { streamXlsRows } from "../src/xls/stream-reader" + +/** Every row a stream yields, collected. */ +export async function streamed( + xls: Uint8Array, + options?: ReadOptions & { sheet?: number | string }, +): Promise { + const out: StreamRow[] = [] + for await (const row of streamXlsRows(xls, options)) out.push(row) + return out +} + +/** + * A dense sheet as a stream yields it: the empty rows left out, each row + * cut after its last value. A value is never null, so trailing nulls are + * only the rectangle's padding. + */ +export function asStreamed(rows: CellValue[][]): StreamRow[] { + const out: StreamRow[] = [] + rows.forEach((row, index) => { + let end = row.length + while (end > 0 && row[end - 1] === null) end-- + if (end > 0) out.push({ index, values: row.slice(0, end) }) + }) + return out +} + +/** Each sheet, streamed, holds exactly what readXls reads for it. */ +export async function expectStreamMatchesReadXls( + xls: Uint8Array, + options?: ReadOptions, +): Promise { + const workbook = await readXls(xls, options) + for (const [sheet, { rows }] of workbook.sheets.entries()) { + expect(await streamed(xls, { ...options, sheet })).toEqual(asStreamed(rows)) + } +} diff --git a/test/exports.test.ts b/test/exports.test.ts index 182f8d2b..6cda9f2e 100644 --- a/test/exports.test.ts +++ b/test/exports.test.ts @@ -149,7 +149,7 @@ describe("hucre/xlsx", () => { }) it("exports the other formats it can read", () => { - expectExports(xlsx, ["readXlsb", "readXls"], "hucre/xlsx") + expectExports(xlsx, ["readXlsb", "readXls", "streamXlsRows"], "hucre/xlsx") }) it("exports the streaming API", () => { diff --git a/test/real-files.test.ts b/test/real-files.test.ts index 7c41d00e..3bc27807 100644 --- a/test/real-files.test.ts +++ b/test/real-files.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest" import { readFileSync } from "node:fs" import { read, readXls, readXlsb, readXlsx, streamXlsxRows } from "../src/index" import type { Cell, CellStyle, ReadOptions, Sheet, Workbook } from "../src/_types" +import { expectStreamMatchesReadXls } from "./_xls-stream" // ═══════════════════════════════════════════════════════════════════════ // #464 — files hucre did not write. @@ -372,6 +373,29 @@ describe("workbooks written by Excel, not by this test suite", () => { // leaking into the model. PARITY.md already says XLS and XLSB carry // less than XLSX — styles, dimensions, properties — so the comparison // is limited to the values, which all three do claim to carry. + // streamXlsRows decodes with readXls's code but walks the records its + // own way — a row at a time, ending each row when the next one starts — + // so files from two producers check it agrees with readXls on every + // value, every row and every sheet. + describe("streamXlsRows streams what readXls reads", () => { + const files = [ + ...FIXTURES.filter((f) => f.reader === readXls).map((f) => f.file), + ...[ + "sheetjs-dates.xls", + "sheetjs-shared-strings.xls", + "sheetjs-sparse.xls", + "sheetjs-unicode.xls", + "sheetjs-values.xls", + "sheetjs-wide.xls", + ].map((file) => `third-party/${file}`), + ] + for (const file of files) { + it(file, async () => { + await expectStreamMatchesReadXls(bytes(file)) + }) + } + }) + describe("one sheet, three containers", () => { const rowsOf = async (file: string, reader: typeof readXlsx): Promise => (await modelOf(file, reader)).sheets[0]?.rows ?? [] diff --git a/test/xls.test.ts b/test/xls.test.ts index 219d1c35..598f9434 100644 --- a/test/xls.test.ts +++ b/test/xls.test.ts @@ -1,7 +1,10 @@ import { describe, expect, it } from "vitest" import { writeCfb } from "../src/xlsx/crypto/cfb" import { readXls } from "../src/xls/reader" +import { streamXlsRows } from "../src/xls/stream-reader" import { read } from "../src/defter" +import { ParseError } from "../src/errors" +import { expectStreamMatchesReadXls, streamed } from "./_xls-stream" // ── Minimal BIFF8 .xls builder (test-only) ─────────────────────────── @@ -52,6 +55,7 @@ const SID = { XF: 0x00e0, BOUNDSHEET: 0x0085, MERGECELLS: 0x00e5, + BLANK: 0x0201, BOF: 0x0809, } @@ -535,3 +539,171 @@ describe("XLS (BIFF5) reader", () => { expect(wb.sheets[0].rows[0][0]).toBe("Видаткова накладна") }) }) + +// ── streamXlsRows ──────────────────────────────────────────────────── +// The stream decodes values with readXls's own code, so most of what +// follows is about shape: which sheet, which rows, in what order, and +// when it stops reading. + +const label = (row: number, col: number, text: string): number[] => + record(SID.LABEL, [...u16(row), ...u16(col), ...u16(0), ...xlStr(text)]) + +/** Two sheets, each holding its own name in A1. */ +function twoSheetXls(): Uint8Array { + const sheets = ["First", "Second"].map((name) => concat([bof(0x0010), label(0, 0, name), eof()])) + const makeGlobals = (firstPos: number): Uint8Array => + concat([ + bof(0x0005), + generalXf(), + record(SID.BOUNDSHEET, [...u32(firstPos), 0, 0, ...shortStr("First")]), + record(SID.BOUNDSHEET, [...u32(firstPos + sheets[0].length), 0, 0, ...shortStr("Second")]), + eof(), + ]) + const globals = makeGlobals(makeGlobals(0).length) + return writeCfb([{ name: "Workbook", data: concat([globals, ...sheets]) }]) +} + +/** Rows 0–4, the cell in column c of row r reading `${r}${c}`; rows 1 and 2 four columns wide. */ +const gridRecords = (): number[][] => [ + label(0, 0, "00"), + ...[0, 1, 2, 3].map((c) => label(1, c, `1${c}`)), + ...[0, 1, 2, 3].map((c) => label(2, c, `2${c}`)), + label(3, 0, "30"), + label(4, 0, "40"), +] + +/** A LABEL record cut off after its row — malformed wherever it sits. */ +const truncatedLabel = (row: number): number[] => record(SID.LABEL, u16(row)) + +describe("streamXlsRows", () => { + describe("streams what readXls reads", () => { + it("BIFF8: SST labels, RK, MULRK, numbers, bools, errors and dates", async () => { + await expectStreamMatchesReadXls(buildXls()) + }) + + it("BIFF8: a FORMAT record that redefines a built-in id", async () => { + await expectStreamMatchesReadXls( + buildXls({ dateFmtId: 50, fmtCodes: [[50, "000000000000"]], value: 81227827687 }), + ) + }) + + it("BIFF8: the 1904 date system, when asked for", async () => { + await expectStreamMatchesReadXls(buildXls(), { dateSystem: "1904" }) + }) + + it("BIFF5: code-page text, RSTRING, and a string formula's STRING record", async () => { + await expectStreamMatchesReadXls(buildXls5({ codepage: 1251 })) + await expectStreamMatchesReadXls(buildXls5(), { codepage: 1251 }) + }) + }) + + it("streams the first sheet, or the one named by index or by name", async () => { + const xls = twoSheetXls() + + expect(await streamed(xls)).toEqual([{ index: 0, values: ["First"] }]) + expect(await streamed(xls, { sheet: 1 })).toEqual([{ index: 0, values: ["Second"] }]) + expect(await streamed(xls, { sheet: "Second" })).toEqual([{ index: 0, values: ["Second"] }]) + }) + + it("yields nothing for a sheet that is not there", async () => { + const xls = twoSheetXls() + + expect(await streamed(xls, { sheet: 2 })).toEqual([]) + expect(await streamed(xls, { sheet: "Third" })).toEqual([]) + }) + + it("skips an empty row and keeps each row's own index", async () => { + // Row 1 carries only a BLANK — formatting, no value — and row 2 nothing at all. + const xls = oneSheetXls( + [generalXf()], + [label(0, 0, "a"), record(SID.BLANK, [...u16(1), ...u16(0), ...u16(0)]), label(3, 2, "d")], + ) + + expect(await streamed(xls)).toEqual([ + { index: 0, values: ["a"] }, + { index: 3, values: [null, null, "d"] }, + ]) + }) + + it("applies range as streamXlsxRows does: rows outside skipped, columns outside masked", async () => { + const xls = oneSheetXls([generalXf()], gridRecords()) + + expect(await streamed(xls, { range: "B2:C3" })).toEqual([ + { index: 1, values: [null, "11", "12", null] }, + { index: 2, values: [null, "21", "22", null] }, + ]) + }) + + it("stops reading once range or maxRows can emit nothing more", async () => { + // Everything after row 4 is damaged: reading that far is an error, so + // a read that succeeds is one that stopped before it. + const xls = oneSheetXls([generalXf()], [...gridRecords(), truncatedLabel(5)]) + + await expect(streamed(xls)).rejects.toThrow(ParseError) + expect((await streamed(xls, { range: "A1:D3" })).map((row) => row.index)).toEqual([0, 1, 2]) + expect((await streamed(xls, { maxRows: 2 })).map((row) => row.index)).toEqual([0, 1]) + }) + + it("buffers a sheet whose cells go back a row, and yields each row once, in order", async () => { + // Streaming such a sheet would have to yield row 0 twice, or split it. + const xls = oneSheetXls( + [generalXf()], + [label(2, 0, "c"), label(0, 0, "a"), label(1, 0, "b"), label(0, 1, "a2")], + ) + + expect(await streamed(xls)).toEqual([ + { index: 0, values: ["a", "a2"] }, + { index: 1, values: ["b"] }, + { index: 2, values: ["c"] }, + ]) + await expectStreamMatchesReadXls(xls) + }) + + it("bounds a buffered sheet by maxTotalCells as readXls bounds it, and a streamed one not at all", async () => { + const backwards = oneSheetXls([generalXf()], [label(2, 1, "c"), label(0, 0, "a")]) + const forwards = oneSheetXls([generalXf()], [label(0, 0, "a"), label(2, 1, "c")]) + + await expect(readXls(backwards, { maxTotalCells: 4 })).rejects.toThrow(/maxTotalCells/) + await expect(streamed(backwards, { maxTotalCells: 4 })).rejects.toThrow(/maxTotalCells/) + // Nothing is held while streaming, so there is nothing for the bound to bound. + expect(await streamed(forwards, { maxTotalCells: 4 })).toHaveLength(2) + }) + + it("reports damage as a ParseError, before the first row and mid-sheet alike", async () => { + await expect(streamed(new Uint8Array([1, 2, 3, 4]))).rejects.toThrow(ParseError) + const xls = oneSheetXls([generalXf()], [label(0, 0, "a"), truncatedLabel(1)]) + await expect(streamed(xls)).rejects.toThrow(ParseError) + }) + + it("refuses a column past the sheet's bounds, as readXls does", async () => { + const xls = oneSheetXls([generalXf()], [label(0, 20_000, "far")]) + + await expect(readXls(xls)).rejects.toThrow(/outside the supported sheet bounds/) + await expect(streamed(xls)).rejects.toThrow(/outside the supported sheet bounds/) + }) + + it("takes a ReadableStream, as the other readers do, bounded by maxInputBytes", async () => { + const bytes = buildXls() + const input = (): ReadableStream => + new ReadableStream({ + start(controller) { + controller.enqueue(bytes) + controller.close() + }, + }) + const rows = [] + for await (const row of streamXlsRows(input())) rows.push(row) + + expect(rows).toEqual(await streamed(bytes)) + await expect(readXls(input(), { maxInputBytes: 64 })).rejects.toThrow(ParseError) + await expect(streamXlsRows(input(), { maxInputBytes: 64 }).next()).rejects.toThrow(ParseError) + }) + + it("can be abandoned after its first row", async () => { + const rows = streamXlsRows(buildXls()) + + expect((await rows.next()).value).toEqual({ index: 0, values: ["Name", "Score"] }) + expect(await rows.return(undefined)).toEqual({ done: true, value: undefined }) + expect(await rows.next()).toEqual({ done: true, value: undefined }) + }) +})