Overhaul for streaming data directly from zip (optimization)

.github/workflows/docs.yml

@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: "3.11"
       - name: Install Python dependencies
         run: |
           pip install sphinx myst-parser pydata-sphinx-theme sphinx-js

.github/workflows/test.yml

@@ -0,0 +1,20 @@
+name: Run Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+      - name: Install dependencies
+        run: npm ci || npm install
+      - name: Run tests
+        run: node test_gs.mjs

README.md

@@ -15,5 +15,3 @@ node bench.mjs dpsv.trx
 ## Documentation
 
 Full documentation is available at (GitHub Pages URL).
-
-

bench.mjs

@@ -7,70 +7,80 @@ import * as streamline from "./streamlineIO.mjs";
 import * as fs from "fs";
 
 async function main() {
-    let argv = process.argv.slice(2);
-    let argc = argv.length;
-    if (argc < 1) {
-        console.log("arguments required: 'node bench.mjs dpsv.trx'")
-        return
-    }
-    //check input filename
-    let fnm = argv[0];
-    if (!fs.existsSync(fnm)) {
-        console.log("Unable to find NIfTI: " + fnm);
-        return
-    }
-    let re = /(?:\.([^.]+))?$/;
-    let ext = re.exec(fnm)[1];
+  let argv = process.argv.slice(2);
+  let argc = argv.length;
+  if (argc < 1) {
+    console.log("arguments required: 'node bench.mjs dpsv.trx'");
+    return;
+  }
+  //check input filename
+  let fnm = argv[0];
+  if (!fs.existsSync(fnm)) {
+    console.log("Unable to find file: " + fnm);
+    return;
+  }
+  let re = /(?:\.([^.]+))?$/;
+  let ext = re.exec(fnm)[1];
+  ext = ext.toUpperCase();
+  if (ext === "GZ") {
+    ext = re.exec(fnm.slice(0, -3))[1]; // img.trk.gz -> img.trk
     ext = ext.toUpperCase();
-    if (ext === 'GZ') {
-      ext = re.exec(fnm.slice(0, -3))[1]; // img.trk.gz -> img.trk
-      ext = ext.toUpperCase();
-    }
-    let obj = [];
-    let d = Date.now();
-    let nrepeats = 11; //11 iterations, ignore first
-    for (let i = 0; i < nrepeats; i++) {
-        if (i == 1) d = Date.now(); //ignore first run for interpretting/disk
-        if (ext === "FIB" || ext === "TT" ||ext === "VTK" || ext === "TCK" || ext === "TRK" || ext === "ZSTD" || ext === "ZST") {
-            const buf = fs.readFileSync(fnm);
-            if (ext === "TCK")
-                obj = streamline.readTCK(new Uint8Array(buf).buffer);
-            else if (ext === "TT")
-                obj = streamline.readTT(new Uint8Array(buf).buffer);
-            else if (ext === "FIB" || ext === "VTK")
-                obj = streamline.readVTK(new Uint8Array(buf).buffer);
-            else
-                obj = streamline.readTRK(new Uint8Array(buf).buffer);
-        } else {
-            obj = await streamline.readTRX(fnm, true);
-        }
-    }
-    let ms = Date.now() - d;
-    //find file size:
-    let dat = fs.readFileSync(fnm);
-    console.log(`${fnm}\tSize\t${dat.length}\tTime\t${ms}`);
-    console.log("Vertices:" + obj.pts.length / 3);
-    console.log(" First vertex (x,y,z):" + obj.pts[0] + ',' + obj.pts[1] + ',' + obj.pts[2]);
-    console.log("Streamlines: " + (obj.offsetPt0.length - 1)); //-1 due to fence post
-    console.log(" Vertices in first streamline: " + (obj.offsetPt0[1] - obj.offsetPt0[0]));
-    if (obj.hasOwnProperty("dpg")) {
-        console.log("dpg (data_per_group) items: " + obj.dpg.length);
-        for (let i = 0; i < obj.dpg.length; i++)
-            console.log("  '" + obj.dpg[i].id + "' items: " + obj.dpg[i].vals.length);
-    }
-    if (obj.hasOwnProperty("dps")) {
-        console.log("dps (data_per_streamline) items: " + obj.dps.length);
-        for (let i = 0; i < obj.dps.length; i++)
-            console.log("  '" + obj.dps[i].id + "' items: " + obj.dps[i].vals.length);
-    }
-    if (obj.hasOwnProperty("dpv")) {
-        console.log("dpv (data_per_vertex) items: " + obj.dpv.length);
-        for (let i = 0; i < obj.dpv.length; i++)
-            console.log("  '" + obj.dpv[i].id + "' items: " + obj.dpv[i].vals.length);
-    }
-    if (obj.hasOwnProperty("header")) {
-        console.log("Header (header.json):");
-        console.log(obj.header);
+  }
+  let obj = [];
+  let d = Date.now();
+  let nrepeats = 11; //11 iterations, ignore first
+  for (let i = 0; i < nrepeats; i++) {
+    if (i == 1) d = Date.now(); //ignore first run for interpretting/disk
+    if (
+      ext === "FIB" ||
+      ext === "TT" ||
+      ext === "VTK" ||
+      ext === "TCK" ||
+      ext === "TRK" ||
+      ext === "ZSTD" ||
+      ext === "ZST"
+    ) {
+      const buf = fs.readFileSync(fnm);
+      if (ext === "TCK") obj = streamline.readTCK(new Uint8Array(buf).buffer);
+      else if (ext === "TT")
+        obj = streamline.readTT(new Uint8Array(buf).buffer);
+      else if (ext === "FIB" || ext === "VTK")
+        obj = streamline.readVTK(new Uint8Array(buf).buffer);
+      else obj = streamline.readTRK(new Uint8Array(buf).buffer);
+    } else {
+      obj = await streamline.readTRX(fnm, true);
     }
+  }
+  let ms = Date.now() - d;
+  //find file size:
+  let dat = fs.readFileSync(fnm);
+  console.log(`${fnm}\tSize\t${dat.length}\tTime\t${ms}`);
+  console.log("Vertices:" + obj.pts.length / 3);
+  console.log(
+    " First vertex (x,y,z):" + obj.pts[0] + "," + obj.pts[1] + "," + obj.pts[2],
+  );
+  console.log("Streamlines: " + (obj.offsetPt0.length - 1)); //-1 due to fence post
+  console.log(
+    " Vertices in first streamline: " + (obj.offsetPt0[1] - obj.offsetPt0[0]),
+  );
+  if (obj.hasOwnProperty("dpg")) {
+    console.log("dpg (data_per_group) items: " + obj.dpg.length);
+    for (let i = 0; i < obj.dpg.length; i++)
+      console.log("  '" + obj.dpg[i].id + "' items: " + obj.dpg[i].vals.length);
+  }
+  if (obj.hasOwnProperty("dps")) {
+    console.log("dps (data_per_streamline) items: " + obj.dps.length);
+    for (let i = 0; i < obj.dps.length; i++)
+      console.log("  '" + obj.dps[i].id + "' items: " + obj.dps[i].vals.length);
+  }
+  if (obj.hasOwnProperty("dpv")) {
+    console.log("dpv (data_per_vertex) items: " + obj.dpv.length);
+    for (let i = 0; i < obj.dpv.length; i++)
+      console.log("  '" + obj.dpv[i].id + "' items: " + obj.dpv[i].vals.length);
+  }
+  if (obj.hasOwnProperty("header")) {
+    console.log("Header (header.json):");
+    console.log(obj.header);
+  }
 }
-main().then(() => console.log('Done'))
+main().then(() => console.log("Done"));

docs/_templates/sidebar-nav-global.html

@@ -1,16 +1,11 @@
-<nav class="bd-docs-nav bd-links"
-     aria-label="{{ _('Section Navigation') }}">
-  <p class="bd-links__title" role="heading" aria-level="1">{{ _("Section Navigation") }}</p>
+<nav class="bd-docs-nav bd-links" aria-label="{{ _('Section Navigation') }}">
+  <p class="bd-links__title" role="heading" aria-level="1">
+    {{ _("Section Navigation") }}
+  </p>
   <div class="bd-toc-item navbar-nav">
-    {{- generate_toctree_html(
-      "sidebar",
-      startdepth=0,
-      show_nav_level=theme_show_nav_level | int,
-      maxdepth=theme_navigation_depth | int,
-      collapse=theme_collapse_navigation | tobool,
-      includehidden=theme_sidebar_includehidden | tobool,
-      titles_only=True
-      )
-    -}}
+    {{- generate_toctree_html( "sidebar", startdepth=0,
+    show_nav_level=theme_show_nav_level | int, maxdepth=theme_navigation_depth |
+    int, collapse=theme_collapse_navigation | tobool,
+    includehidden=theme_sidebar_includehidden | tobool, titles_only=True ) -}}
   </div>
 </nav>

docs/api.md

@@ -2,38 +2,43 @@
 
 All reader functions return an object with the following structure:
 
-| Field | Type | Description |
-|-------|------|-------------|
-| `pts` | `Float32Array` | Interleaved XYZ vertex positions for all streamlines |
-| `offsetPt0` | `Uint32Array` | Start index of each streamline in `pts` (NB_STREAMLINES + 1 entries) |
-| `dps` | `Array<{id, vals}>` | Data-per-streamline (TRX, TRK) |
-| `dpv` | `Array<{id, vals}>` | Data-per-vertex (TRX, TRK) |
-| `dpg` | `Array<{id, vals}>` | Data-per-group (TRX only) |
-| `header` | `Object` | Parsed `header.json` (TRX only) |
+| Field       | Type                | Description                                                          |
+| ----------- | ------------------- | -------------------------------------------------------------------- |
+| `pts`       | `Float32Array`      | Interleaved XYZ vertex positions for all streamlines                 |
+| `offsetPt0` | `Uint32Array`       | Start index of each streamline in `pts` (NB_STREAMLINES + 1 entries) |
+| `dps`       | `Array<{id, vals}>` | Data-per-streamline (TRX, TRK)                                       |
+| `dpv`       | `Array<{id, vals}>` | Data-per-vertex (TRX, TRK)                                           |
+| `dpg`       | `Array<{id, vals}>` | Data-per-group (TRX only)                                            |
+| `header`    | `Object`            | Parsed `header.json` (TRX only)                                      |
 
 ## Reader Functions
 
 ### readTRK
 
 ```{js:autofunction} readTRK
+
 ```
 
 ### readTCK
 
 ```{js:autofunction} readTCK
+
 ```
 
 ### readVTK
 
 ```{js:autofunction} readVTK
+
 ```
 
 ### readTRX
 
 ```{js:autofunction} readTRX
+
 ```
 
 ### readTT
 
 ```{js:autofunction} readTT
+
 ```

docs/benchmark.md

@@ -24,19 +24,19 @@ Benchmark run on a MacBook with an M2 Pro CPU, loading from local SSD.
 
 ### Format Summary
 
-| Format | Extension | Notes |
-|--------|-----------|-------|
-| tt | `.tt` | DSI Studio — very compact; 1/32 voxel precision (slightly lossy) |
-| tt.gz | `.tt.gz` | Gzip-compressed DSI Studio |
-| vtk | `.vtk` | VTK legacy (DiPy OFFSETS, `--offsets_dtype uint32`) |
-| tck | `.tck` | MRtrix format |
-| trk | `.trk` | TrackVis format |
-| trk.gz | `.trk.gz` | Gzip-compressed TrackVis |
-| trk.zst | `.trk.zst` | Zstandard-compressed TrackVis |
-| trx | `.trx` | Uncompressed TRX |
-| z.trx | `.trx` | Zip-compressed TRX |
-| 16.trx | `.trx` | TRX with float16 positions |
-| 16z.trx | `.trx` | Zip-compressed TRX with float16 positions |
+| Format  | Extension  | Notes                                                            |
+| ------- | ---------- | ---------------------------------------------------------------- |
+| tt      | `.tt`      | DSI Studio — very compact; 1/32 voxel precision (slightly lossy) |
+| tt.gz   | `.tt.gz`   | Gzip-compressed DSI Studio                                       |
+| vtk     | `.vtk`     | VTK legacy (DiPy OFFSETS, `--offsets_dtype uint32`)              |
+| tck     | `.tck`     | MRtrix format                                                    |
+| trk     | `.trk`     | TrackVis format                                                  |
+| trk.gz  | `.trk.gz`  | Gzip-compressed TrackVis                                         |
+| trk.zst | `.trk.zst` | Zstandard-compressed TrackVis                                    |
+| trx     | `.trx`     | Uncompressed TRX                                                 |
+| z.trx   | `.trx`     | Zip-compressed TRX                                               |
+| 16.trx  | `.trx`     | TRX with float16 positions                                       |
+| 16z.trx | `.trx`     | Zip-compressed TRX with float16 positions                        |
 
 > Note: Native zstd decompression is very fast, but the JavaScript
 > decompressor (`fzstd`) is relatively slower than native implementations.

docs/getting-started.md

@@ -28,15 +28,15 @@ node bench.mjs dpsv.trx
 ### In Your Code
 
 ```javascript
-import { readTRK, readTCK, readVTK, readTRX, readTT } from './streamlineIO.mjs';
-import * as fs from 'fs';
+import { readTRK, readTCK, readVTK, readTRX, readTT } from "./streamlineIO.mjs";
+import * as fs from "fs";
 
 // TRK, TCK, VTK, TT — synchronous, from ArrayBuffer
-const trkData = readTRK(fs.readFileSync('tracts.trk').buffer);
+const trkData = readTRK(fs.readFileSync("tracts.trk").buffer);
 console.log(`${trkData.offsetPt0.length - 1} streamlines`);
 
 // TRX — async (supports URL or local file)
-const trxData = await readTRX('tracts.trx');
+const trxData = await readTRX("tracts.trx");
 console.log(trxData.header);
 ```