diff --git a/src/xls/biff.ts b/src/xls/biff.ts index 763cc950..f2a0f53c 100644 --- a/src/xls/biff.ts +++ b/src/xls/biff.ts @@ -45,22 +45,74 @@ export const SID = { BOF: 0x0809, } as const -/** Parse a stream into records, recording each one's byte offset. */ -export function parseRecords(stream: Uint8Array): BiffRecord[] { - const out: BiffRecord[] = [] - const view = new DataView(stream.buffer, stream.byteOffset, stream.byteLength) - let pos = 0 - while (pos + 4 <= stream.length) { - const id = view.getUint16(pos, true) - const len = view.getUint16(pos + 2, true) - const start = pos - pos += 4 - out.push({ id, data: stream.subarray(pos, pos + len), offset: start }) - pos += len +/** + * Walks a stream's records in place, one at a time. + * + * The reader used to materialize every record before decoding the first: + * an object and a view apiece, ~155 bytes each, and a map from offset to + * index on top. A 17 MB workbook is ~960k records, so that was ~150 MB + * held for the whole read. Framing is unchanged — a record running past + * the end of the stream is cut short, and a zero id with a zero length is + * padding that ends it. + */ +export class RecordCursor { + private view: DataView + private ended = false + + constructor( + private stream: Uint8Array, + private pos = 0, + ) { + this.view = new DataView(stream.buffer, stream.byteOffset, stream.byteLength) + } + + /** Move past the record at the cursor and return its offset, or -1 at the end. */ + skip(): number { + if (this.ended || this.pos + 4 > this.stream.length) return -1 + const offset = this.pos + const id = this.view.getUint16(offset, true) + const len = this.view.getUint16(offset + 2, true) + this.pos = offset + 4 + len // A zero id past real data means padding — stop. - if (id === 0 && len === 0) break + if (id === 0 && len === 0) this.ended = true + return offset + } + + next(): BiffRecord | undefined { + const offset = this.skip() + if (offset < 0) return undefined + const len = this.view.getUint16(offset + 2, true) + return { + id: this.view.getUint16(offset, true), + data: this.stream.subarray(offset + 4, offset + 4 + len), + offset, + } + } + + /** The record `next` would return, without moving past it. */ + peek(): BiffRecord | undefined { + const { pos, ended } = this + const rec = this.next() + this.pos = pos + this.ended = ended + return rec } - return out +} + +/** + * Which of `offsets` begin a record. A BOUNDSHEET names its sheet by the + * byte offset of the sheet's BOF, and one landing inside a record — a + * damaged file — has always read as an empty sheet rather than as records + * taken out of phase. + */ +export function recordStarts(stream: Uint8Array, offsets: Iterable): Set { + const wanted = new Set(offsets) + const found = new Set() + const cursor = new RecordCursor(stream) + for (let at = cursor.skip(); at >= 0 && found.size < wanted.size; at = cursor.skip()) { + if (wanted.has(at)) found.add(at) + } + return found } /** Little-endian cursor over a record body. */ @@ -103,6 +155,43 @@ export class Reader { } } +/** The most characters handed to one `String.fromCharCode` call. */ +const CHARS_PER_CALL = 8192 + +/** + * `count` characters from `start`: a byte each when compressed (the byte + * is the code point), else UTF-16LE code units. The caller guarantees the + * bytes are there. + * + * Built a character at a time with `+=`, a string is a rope, one node per + * character until something flattens it, and nothing in a read did. A + * workbook's 2.3 million shared characters held ~70 MB that way, against + * ~7 MB decoded flat. TextDecoder would be wrong twice over: the Encoding + * Standard maps "latin1" to windows-1252, which moves 0x80–0x9F, and + * "utf-16le" turns a lone surrogate into U+FFFD — which is what each half + * of a pair becomes when a CONTINUE boundary splits it. + */ +export function decodeChars( + buf: Uint8Array, + start: number, + count: number, + compressed: boolean, +): string { + const units = compressed ? buf.subarray(start, start + count) : utf16Units(buf, start, count) + if (units.length <= CHARS_PER_CALL) return String.fromCharCode(...units) + const parts: string[] = [] + for (let i = 0; i < units.length; i += CHARS_PER_CALL) { + parts.push(String.fromCharCode(...units.subarray(i, i + CHARS_PER_CALL))) + } + return parts.join("") +} + +function utf16Units(buf: Uint8Array, start: number, count: number): Uint16Array { + const units = new Uint16Array(count) + for (let i = 0, p = start; i < count; i++, p += 2) units[i] = buf[p] | (buf[p + 1] << 8) + return units +} + /** Decode an RK number (MS-XLS §2.5.166). */ export function decodeRk(rk: number): number { const fX100 = (rk & 1) !== 0 @@ -171,6 +260,12 @@ class BlockStream { this.pos += 4 return v } + /** `count` characters from the current block, which holds them all. */ + chars(count: number, compressed: boolean): string { + const s = decodeChars(this.cur(), this.pos, count, compressed) + this.pos += compressed ? count : count * 2 + return s + } skip(n: number): void { // skip may cross blocks (rich/phonetic trailers), no grbit byte let left = n @@ -224,6 +319,7 @@ function readSstString(s: BlockStream): string { const cRun = rich ? s.u16() : 0 const cbExt = phonetic ? s.u32() : 0 + // A block's worth of characters at a time — see decodeChars. let str = "" let read = 0 while (read < cch) { @@ -234,8 +330,17 @@ function readSstString(s: BlockStream): string { grbit = s.u8() compressed = (grbit & 0x01) === 0 } - str += String.fromCharCode(compressed ? s.u8() : s.u16()) - read++ + const fit = compressed ? s.remainingInBlock() : s.remainingInBlock() >> 1 + if (fit > 0) { + const take = Math.min(cch - read, fit) + str += s.chars(take, compressed) + read += take + } else { + // A two-byte character straddling the end of a block is malformed; + // it reads exactly as it always has. + str += String.fromCharCode(compressed ? s.u8() : s.u16()) + read++ + } } if (rich) s.skip(cRun * 4) if (phonetic) s.skip(cbExt) diff --git a/src/xls/reader.ts b/src/xls/reader.ts index feaeceda..634c04ac 100644 --- a/src/xls/reader.ts +++ b/src/xls/reader.ts @@ -11,7 +11,16 @@ import { MAX_COL_INDEX, MAX_ROW_INDEX, MAX_TOTAL_CELLS } from "../limits" import { readInputToUint8Array } from "../_input" import { readCfb } from "../xlsx/crypto/cfb" import { isBuiltinDateFormatId, isDateFormat, serialToDate } from "../_date" -import { decodeRk, parseRecords, parseSst, Reader, SID, type BiffRecord } from "./biff" +import { + decodeChars, + decodeRk, + parseSst, + Reader, + RecordCursor, + recordStarts, + SID, + type BiffRecord, +} from "./biff" import { codepageDecoder } from "./codepage" const ERROR_TEXT: Record = { @@ -106,8 +115,9 @@ function biff5Strings(decode: (bytes: Uint8Array) => string): BiffStrings { * of the globals pass because the strings that pass decodes — sheet names, * format codes — may in principle precede it. */ -function findCodepage(records: BiffRecord[]): number | undefined { - for (const rec of records) { +function findCodepage(stream: Uint8Array): number | undefined { + const cursor = new RecordCursor(stream) + for (let rec = cursor.next(); rec; rec = cursor.next()) { if (rec.id === SID.EOF) return undefined if (rec.id === SID.CODEPAGE) { if (rec.data.length < 2) throw new ParseError("Invalid XLS: CODEPAGE record is too short") @@ -118,7 +128,7 @@ function findCodepage(records: BiffRecord[]): number | undefined { } function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbook { - const records = parseRecords(stream) + const cursor = new RecordCursor(stream) // ── BIFF version gate ── // The first record is the workbook globals BOF; its first u16 is the BIFF @@ -126,7 +136,7 @@ function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbo // are Excel 5.0 and 95 — the same record layout except that strings are // code-page bytes and there is no shared-string table. Anything else is // rejected rather than misread. - const bof = records[0] + const bof = cursor.next() if (!bof || bof.id !== SID.BOF) { throw new ParseError("Invalid XLS: missing BOF record at start of Workbook stream") } @@ -145,28 +155,18 @@ function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbo // any string is read, so a bad page fails here rather than mid-sheet. const strings: BiffStrings = biffVersion === BIFF5 - ? biff5Strings( - codepageDecoder(findCodepage(records) ?? options?.codepage ?? DEFAULT_CODEPAGE), - ) + ? biff5Strings(codepageDecoder(findCodepage(stream) ?? options?.codepage ?? DEFAULT_CODEPAGE)) : BIFF8_STRINGS - const offsetToIndex = new Map() - for (let i = 0; i < records.length; i++) offsetToIndex.set(records[i].offset, i) - - // ── Globals substream (records[0] = BOF … first EOF) ── + // ── Globals substream (BOF … first EOF) ── let date1904 = options?.dateSystem === "1904" const xfFmtIds: number[] = [] const fmtCodes = new Map() const boundSheets: Array<{ name: string; pos: number }> = [] const sst: string[] = [] - let gi = 0 - for (; gi < records.length; gi++) { - const rec = records[gi] - if (rec.id === SID.EOF) { - gi++ - break - } + for (let rec: BiffRecord | undefined = bof; rec; rec = cursor.next()) { + if (rec.id === SID.EOF) break switch (rec.id) { case SID.DATEMODE: { if (!options?.dateSystem || options.dateSystem === "auto") { @@ -199,9 +199,9 @@ function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbo case SID.SST: { const blocks: Uint8Array[] = [rec.data] // Gather trailing CONTINUE records belonging to the SST. - for (let j = gi + 1; j < records.length; j++) { - if (records[j].id !== SID.CONTINUE) break - blocks.push(records[j].data) + for (let next = cursor.peek(); next?.id === SID.CONTINUE; next = cursor.peek()) { + blocks.push(next.data) + cursor.next() } // A loop, not `push(...parseSst(blocks))`: spreading an array as // arguments puts one stack slot per element, so a workbook with a @@ -233,17 +233,19 @@ function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbo const isDate = (ixfe: number): boolean => dateXf[ixfe] === true // ── Sheet substreams ── + const starts = recordStarts( + stream, + boundSheets.map((bs) => bs.pos), + ) const sheets: Sheet[] = [] for (const bs of boundSheets) { - const startIdx = offsetToIndex.get(bs.pos) - if (startIdx === undefined) { + if (!starts.has(bs.pos)) { sheets.push({ name: bs.name, rows: [] }) continue } sheets.push( parseSheet( - records, - startIdx, + new RecordCursor(stream, bs.pos), bs.name, sst, strings, @@ -258,8 +260,7 @@ function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbo } function parseSheet( - records: BiffRecord[], - startIdx: number, + cursor: RecordCursor, name: string, sst: string[], strings: BiffStrings, @@ -304,8 +305,8 @@ function parseSheet( setCell(row, col, isDate(ixfe) ? serialToDate(n, date1904) : n) } - for (let i = startIdx + 1; i < records.length; i++) { - const rec = records[i] + cursor.skip() // the sheet's BOF + for (let rec = cursor.next(); rec; rec = cursor.next()) { if (rec.id === SID.EOF) break const r = new Reader(rec.data) switch (rec.id) { @@ -374,7 +375,7 @@ function parseSheet( setCell(row, col, ERROR_TEXT[b[2]] ?? "#ERR!") // error else if (kind === 0) { // string: value is in the following STRING record - const next = records[i + 1] + const next = cursor.peek() if (next && next.id === SID.STRING) setCell(row, col, strings.long(new Reader(next.data))) } @@ -449,6 +450,15 @@ function readShortString(r: Reader): string { function readChars(r: Reader, cch: number): string { const grbit = r.u8() const compressed = (grbit & 0x01) === 0 + const bytes = compressed ? cch : cch * 2 + if (bytes <= r.remaining()) { + const s = decodeChars(r.buf, r.pos, cch, compressed) + r.skip(bytes) + return s + } + // A string running past its record — truncated, or a long formula result + // continued into a CONTINUE record, which is not followed here — reads + // as it always has, a character at a time. let s = "" for (let i = 0; i < cch; i++) s += String.fromCharCode(compressed ? r.u8() : r.u16()) return s diff --git a/test/xls.test.ts b/test/xls.test.ts index fb702df0..219d1c35 100644 --- a/test/xls.test.ts +++ b/test/xls.test.ts @@ -39,6 +39,7 @@ const rkInt = (v: number): number[] => u32(((v << 2) | 2) >>> 0) const SID = { FORMULA: 0x0006, EOF: 0x000a, + CONTINUE: 0x003c, DATEMODE: 0x0022, FORMAT: 0x041e, NUMBER: 0x0203, @@ -231,6 +232,113 @@ describe("XLS (BIFF8) reader", () => { }) }) +// ── Strings decoded a run at a time ────────────────────────────────── +// Text is decoded a record's worth of characters at a time, not one by +// one. Excel splits a long shared string wherever its record fills, and +// each CONTINUE that resumes one restarts with its own flag byte, so a +// string can change width mid-way and a surrogate pair can be cut in two. +// These pass on main as well: they pin what the faster path must keep. + +const latin = (s: string): number[] => [...s].map((c) => c.charCodeAt(0)) +function utf16(s: string): number[] { + const out: number[] = [] + for (let i = 0; i < s.length; i++) out.push(...u16(s.charCodeAt(i))) + return out +} +const generalXf = (): number[] => + record(SID.XF, [...u16(0), ...u16(0), ...Array.from({ length: 16 }, () => 0)]) + +/** One sheet after the given globals records, its BOUNDSHEET pointing at it. */ +function oneSheetXls(globals: number[][], sheet: number[][], sheetName = "Sheet1"): Uint8Array { + const makeGlobals = (sheetPos: number): Uint8Array => + concat([ + bof(0x0005), + ...globals, + record(SID.BOUNDSHEET, [...u32(sheetPos), 0, 0, ...shortStr(sheetName)]), + eof(), + ]) + const globalsStream = makeGlobals(makeGlobals(0).length) + return writeCfb([ + { name: "Workbook", data: concat([globalsStream, bof(0x0010), ...sheet, eof()]) }, + ]) +} + +describe("XLS (BIFF8) strings", () => { + it("joins a shared string's runs across CONTINUE records, whatever their width", async () => { + // 100 one-byte characters in the SST record, a CONTINUE record full of + // them (more than one String.fromCharCode call takes), then a switch + // to UTF-16 for the rest. The next string's surrogate pair straddles + // the following boundary. + const head = "A".repeat(100) + const middle = "B".repeat(8223) + const tail = "Ж".repeat(1677) + const emoji = "ab😀cd" + const xls = oneSheetXls( + [ + generalXf(), + record(SID.SST, [...u32(3), ...u32(3), ...u16(10_000), 0, ...latin(head)]), + record(SID.CONTINUE, [0, ...latin(middle)]), + record(SID.CONTINUE, [ + 1, + ...utf16(tail), + ...u16(emoji.length), + 1, + ...utf16(emoji.slice(0, 3)), + ]), + record(SID.CONTINUE, [1, ...utf16(emoji.slice(3)), ...u16(3), 0, ...latin("end")]), + ], + [0, 1, 2].map((i) => record(SID.LABELSST, [...u16(0), ...u16(i), ...u16(0), ...u32(i)])), + ) + + const wb = await readXls(xls) + + expect(wb.sheets[0].rows[0]).toEqual([head + middle + tail, emoji, "end"]) + }) + + it("reads inline text in either width, longer than one String.fromCharCode call takes", async () => { + const long = "x".repeat(8200) + const xls = oneSheetXls( + [generalXf()], + [ + record(SID.LABEL, [...u16(0), ...u16(0), ...u16(0), ...xlStr(long)]), + record(SID.LABEL, [...u16(0), ...u16(1), ...u16(0), ...u16(5), 1, ...utf16("Ωmega")]), + ], + ) + + const wb = await readXls(xls) + + expect(wb.sheets[0].rows[0]).toEqual([long, "Ωmega"]) + }) +}) + +describe("XLS (BIFF8) sheet offsets", () => { + it("reads a sheet whose BOUNDSHEET offset misses a record boundary as empty", async () => { + // The offset is the only link from a BOUNDSHEET to its sheet. Records + // are walked in place now rather than indexed by offset, and one that + // lands inside a record still means no rows, not records out of phase. + const makeGlobals = (sheetPos: number): Uint8Array => + concat([ + bof(0x0005), + generalXf(), + record(SID.BOUNDSHEET, [...u32(sheetPos), 0, 0, ...shortStr("Good")]), + record(SID.BOUNDSHEET, [...u32(sheetPos + 2), 0, 0, ...shortStr("Off")]), + eof(), + ]) + const globals = makeGlobals(makeGlobals(0).length) + const sheet = concat([ + bof(0x0010), + record(SID.LABEL, [...u16(0), ...u16(0), ...u16(0), ...xlStr("Hi")]), + eof(), + ]) + + const wb = await readXls(writeCfb([{ name: "Workbook", data: concat([globals, sheet]) }])) + + expect(wb.sheets.map((s) => s.name)).toEqual(["Good", "Off"]) + expect(wb.sheets[0].rows).toEqual([["Hi"]]) + expect(wb.sheets[1].rows).toEqual([]) + }) +}) + // ── Minimal BIFF5 .xls builder (test-only) ─────────────────────────── // Excel 5.0/95: the same records, but every string is bytes in the // workbook's code page with no per-string flag, cell text is written