perf(xls): read records in place and decode strings a run at a time

src/xls/biff.ts

@@ -45,22 +45,74 @@ export const SID = {
   BOF: 0x0809,
 } as const
 
-/** Parse a stream into records, recording each one's byte offset. */
-export function parseRecords(stream: Uint8Array): BiffRecord[] {
-  const out: BiffRecord[] = []
-  const view = new DataView(stream.buffer, stream.byteOffset, stream.byteLength)
-  let pos = 0
-  while (pos + 4 <= stream.length) {
-    const id = view.getUint16(pos, true)
-    const len = view.getUint16(pos + 2, true)
-    const start = pos
-    pos += 4
-    out.push({ id, data: stream.subarray(pos, pos + len), offset: start })
-    pos += len
+/**
+ * Walks a stream's records in place, one at a time.
+ *
+ * The reader used to materialize every record before decoding the first:
+ * an object and a view apiece, ~155 bytes each, and a map from offset to
+ * index on top. A 17 MB workbook is ~960k records, so that was ~150 MB
+ * held for the whole read. Framing is unchanged — a record running past
+ * the end of the stream is cut short, and a zero id with a zero length is
+ * padding that ends it.
+ */
+export class RecordCursor {
+  private view: DataView
+  private ended = false
+
+  constructor(
+    private stream: Uint8Array,
+    private pos = 0,
+  ) {
+    this.view = new DataView(stream.buffer, stream.byteOffset, stream.byteLength)
+  }
+
+  /** Move past the record at the cursor and return its offset, or -1 at the end. */
+  skip(): number {
+    if (this.ended || this.pos + 4 > this.stream.length) return -1
+    const offset = this.pos
+    const id = this.view.getUint16(offset, true)
+    const len = this.view.getUint16(offset + 2, true)
+    this.pos = offset + 4 + len
     // A zero id past real data means padding — stop.
-    if (id === 0 && len === 0) break
+    if (id === 0 && len === 0) this.ended = true
+    return offset
+  }
+
+  next(): BiffRecord | undefined {
+    const offset = this.skip()
+    if (offset < 0) return undefined
+    const len = this.view.getUint16(offset + 2, true)
+    return {
+      id: this.view.getUint16(offset, true),
+      data: this.stream.subarray(offset + 4, offset + 4 + len),
+      offset,
+    }
+  }
+
+  /** The record `next` would return, without moving past it. */
+  peek(): BiffRecord | undefined {
+    const { pos, ended } = this
+    const rec = this.next()
+    this.pos = pos
+    this.ended = ended
+    return rec
   }
-  return out
+}
+
+/**
+ * Which of `offsets` begin a record. A BOUNDSHEET names its sheet by the
+ * byte offset of the sheet's BOF, and one landing inside a record — a
+ * damaged file — has always read as an empty sheet rather than as records
+ * taken out of phase.
+ */
+export function recordStarts(stream: Uint8Array, offsets: Iterable<number>): Set<number> {
+  const wanted = new Set(offsets)
+  const found = new Set<number>()
+  const cursor = new RecordCursor(stream)
+  for (let at = cursor.skip(); at >= 0 && found.size < wanted.size; at = cursor.skip()) {
+    if (wanted.has(at)) found.add(at)
+  }
+  return found
 }
 
 /** Little-endian cursor over a record body. */
@@ -103,6 +155,43 @@ export class Reader {
   }
 }
 
+/** The most characters handed to one `String.fromCharCode` call. */
+const CHARS_PER_CALL = 8192
+
+/**
+ * `count` characters from `start`: a byte each when compressed (the byte
+ * is the code point), else UTF-16LE code units. The caller guarantees the
+ * bytes are there.
+ *
+ * Built a character at a time with `+=`, a string is a rope, one node per
+ * character until something flattens it, and nothing in a read did. A
+ * workbook's 2.3 million shared characters held ~70 MB that way, against
+ * ~7 MB decoded flat. TextDecoder would be wrong twice over: the Encoding
+ * Standard maps "latin1" to windows-1252, which moves 0x80–0x9F, and
+ * "utf-16le" turns a lone surrogate into U+FFFD — which is what each half
+ * of a pair becomes when a CONTINUE boundary splits it.
+ */
+export function decodeChars(
+  buf: Uint8Array,
+  start: number,
+  count: number,
+  compressed: boolean,
+): string {
+  const units = compressed ? buf.subarray(start, start + count) : utf16Units(buf, start, count)
+  if (units.length <= CHARS_PER_CALL) return String.fromCharCode(...units)
+  const parts: string[] = []
+  for (let i = 0; i < units.length; i += CHARS_PER_CALL) {
+    parts.push(String.fromCharCode(...units.subarray(i, i + CHARS_PER_CALL)))
+  }
+  return parts.join("")
+}
+
+function utf16Units(buf: Uint8Array, start: number, count: number): Uint16Array {
+  const units = new Uint16Array(count)
+  for (let i = 0, p = start; i < count; i++, p += 2) units[i] = buf[p] | (buf[p + 1] << 8)
+  return units
+}
+
 /** Decode an RK number (MS-XLS §2.5.166). */
 export function decodeRk(rk: number): number {
   const fX100 = (rk & 1) !== 0
@@ -171,6 +260,12 @@ class BlockStream {
     this.pos += 4
     return v
   }
+  /** `count` characters from the current block, which holds them all. */
+  chars(count: number, compressed: boolean): string {
+    const s = decodeChars(this.cur(), this.pos, count, compressed)
+    this.pos += compressed ? count : count * 2
+    return s
+  }
   skip(n: number): void {
     // skip may cross blocks (rich/phonetic trailers), no grbit byte
     let left = n
@@ -224,6 +319,7 @@ function readSstString(s: BlockStream): string {
   const cRun = rich ? s.u16() : 0
   const cbExt = phonetic ? s.u32() : 0
 
+  // A block's worth of characters at a time — see decodeChars.
   let str = ""
   let read = 0
   while (read < cch) {
@@ -234,8 +330,17 @@ function readSstString(s: BlockStream): string {
       grbit = s.u8()
       compressed = (grbit & 0x01) === 0
     }
-    str += String.fromCharCode(compressed ? s.u8() : s.u16())
-    read++
+    const fit = compressed ? s.remainingInBlock() : s.remainingInBlock() >> 1
+    if (fit > 0) {
+      const take = Math.min(cch - read, fit)
+      str += s.chars(take, compressed)
+      read += take
+    } else {
+      // A two-byte character straddling the end of a block is malformed;
+      // it reads exactly as it always has.
+      str += String.fromCharCode(compressed ? s.u8() : s.u16())
+      read++
+    }
   }
   if (rich) s.skip(cRun * 4)
   if (phonetic) s.skip(cbExt)

src/xls/reader.ts

@@ -11,7 +11,16 @@ import { MAX_COL_INDEX, MAX_ROW_INDEX, MAX_TOTAL_CELLS } from "../limits"
 import { readInputToUint8Array } from "../_input"
 import { readCfb } from "../xlsx/crypto/cfb"
 import { isBuiltinDateFormatId, isDateFormat, serialToDate } from "../_date"
-import { decodeRk, parseRecords, parseSst, Reader, SID, type BiffRecord } from "./biff"
+import {
+  decodeChars,
+  decodeRk,
+  parseSst,
+  Reader,
+  RecordCursor,
+  recordStarts,
+  SID,
+  type BiffRecord,
+} from "./biff"
 import { codepageDecoder } from "./codepage"
 
 const ERROR_TEXT: Record<number, string> = {
@@ -106,8 +115,9 @@ function biff5Strings(decode: (bytes: Uint8Array) => string): BiffStrings {
  * of the globals pass because the strings that pass decodes — sheet names,
  * format codes — may in principle precede it.
  */
-function findCodepage(records: BiffRecord[]): number | undefined {
-  for (const rec of records) {
+function findCodepage(stream: Uint8Array): number | undefined {
+  const cursor = new RecordCursor(stream)
+  for (let rec = cursor.next(); rec; rec = cursor.next()) {
     if (rec.id === SID.EOF) return undefined
     if (rec.id === SID.CODEPAGE) {
       if (rec.data.length < 2) throw new ParseError("Invalid XLS: CODEPAGE record is too short")
@@ -118,15 +128,15 @@ function findCodepage(records: BiffRecord[]): number | undefined {
 }
 
 function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbook {
-  const records = parseRecords(stream)
+  const cursor = new RecordCursor(stream)
 
   // ── BIFF version gate ──
   // The first record is the workbook globals BOF; its first u16 is the BIFF
   // version. BIFF8 (0x0600) is Excel 97-2003; BIFF5 and BIFF7 (both 0x0500)
   // are Excel 5.0 and 95 — the same record layout except that strings are
   // code-page bytes and there is no shared-string table. Anything else is
   // rejected rather than misread.
-  const bof = records[0]
+  const bof = cursor.next()
   if (!bof || bof.id !== SID.BOF) {
     throw new ParseError("Invalid XLS: missing BOF record at start of Workbook stream")
   }
@@ -145,28 +155,18 @@ function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbo
   // any string is read, so a bad page fails here rather than mid-sheet.
   const strings: BiffStrings =
     biffVersion === BIFF5
-      ? biff5Strings(
-          codepageDecoder(findCodepage(records) ?? options?.codepage ?? DEFAULT_CODEPAGE),
-        )
+      ? biff5Strings(codepageDecoder(findCodepage(stream) ?? options?.codepage ?? DEFAULT_CODEPAGE))
       : BIFF8_STRINGS
 
-  const offsetToIndex = new Map<number, number>()
-  for (let i = 0; i < records.length; i++) offsetToIndex.set(records[i].offset, i)
-
-  // ── Globals substream (records[0] = BOF … first EOF) ──
+  // ── Globals substream (BOF … first EOF) ──
   let date1904 = options?.dateSystem === "1904"
   const xfFmtIds: number[] = []
   const fmtCodes = new Map<number, string>()
   const boundSheets: Array<{ name: string; pos: number }> = []
   const sst: string[] = []
 
-  let gi = 0
-  for (; gi < records.length; gi++) {
-    const rec = records[gi]
-    if (rec.id === SID.EOF) {
-      gi++
-      break
-    }
+  for (let rec: BiffRecord | undefined = bof; rec; rec = cursor.next()) {
+    if (rec.id === SID.EOF) break
     switch (rec.id) {
       case SID.DATEMODE: {
         if (!options?.dateSystem || options.dateSystem === "auto") {
@@ -199,9 +199,9 @@ function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbo
       case SID.SST: {
         const blocks: Uint8Array[] = [rec.data]
         // Gather trailing CONTINUE records belonging to the SST.
-        for (let j = gi + 1; j < records.length; j++) {
-          if (records[j].id !== SID.CONTINUE) break
-          blocks.push(records[j].data)
+        for (let next = cursor.peek(); next?.id === SID.CONTINUE; next = cursor.peek()) {
+          blocks.push(next.data)
+          cursor.next()
         }
         // A loop, not `push(...parseSst(blocks))`: spreading an array as
         // arguments puts one stack slot per element, so a workbook with a
@@ -233,17 +233,19 @@ function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbo
   const isDate = (ixfe: number): boolean => dateXf[ixfe] === true
 
   // ── Sheet substreams ──
+  const starts = recordStarts(
+    stream,
+    boundSheets.map((bs) => bs.pos),
+  )
   const sheets: Sheet[] = []
   for (const bs of boundSheets) {
-    const startIdx = offsetToIndex.get(bs.pos)
-    if (startIdx === undefined) {
+    if (!starts.has(bs.pos)) {
       sheets.push({ name: bs.name, rows: [] })
       continue
     }
     sheets.push(
       parseSheet(
-        records,
-        startIdx,
+        new RecordCursor(stream, bs.pos),
         bs.name,
         sst,
         strings,
@@ -258,8 +260,7 @@ function parseWorkbookRecords(stream: Uint8Array, options?: ReadOptions): Workbo
 }
 
 function parseSheet(
-  records: BiffRecord[],
-  startIdx: number,
+  cursor: RecordCursor,
   name: string,
   sst: string[],
   strings: BiffStrings,
@@ -304,8 +305,8 @@ function parseSheet(
     setCell(row, col, isDate(ixfe) ? serialToDate(n, date1904) : n)
   }
 
-  for (let i = startIdx + 1; i < records.length; i++) {
-    const rec = records[i]
+  cursor.skip() // the sheet's BOF
+  for (let rec = cursor.next(); rec; rec = cursor.next()) {
     if (rec.id === SID.EOF) break
     const r = new Reader(rec.data)
     switch (rec.id) {
@@ -374,7 +375,7 @@ function parseSheet(
             setCell(row, col, ERROR_TEXT[b[2]] ?? "#ERR!") // error
           else if (kind === 0) {
             // string: value is in the following STRING record
-            const next = records[i + 1]
+            const next = cursor.peek()
             if (next && next.id === SID.STRING)
               setCell(row, col, strings.long(new Reader(next.data)))
           }
@@ -449,6 +450,15 @@ function readShortString(r: Reader): string {
 function readChars(r: Reader, cch: number): string {
   const grbit = r.u8()
   const compressed = (grbit & 0x01) === 0
+  const bytes = compressed ? cch : cch * 2
+  if (bytes <= r.remaining()) {
+    const s = decodeChars(r.buf, r.pos, cch, compressed)
+    r.skip(bytes)
+    return s
+  }
+  // A string running past its record — truncated, or a long formula result
+  // continued into a CONTINUE record, which is not followed here — reads
+  // as it always has, a character at a time.
   let s = ""
   for (let i = 0; i < cch; i++) s += String.fromCharCode(compressed ? r.u8() : r.u16())
   return s