From 1f82a8c3dc482fa1cfe8d60ff2790bb5e6cc68e3 Mon Sep 17 00:00:00 2001 From: Yehonal Date: Mon, 26 Oct 2026 14:49:47 +0000 Subject: [PATCH] fix: batch catalog scans and defer status aggregates --- docs/host-file-catalog.md | 5 +- scripts/test-host-admin-server.mjs | 33 ++++++++++- scripts/test-host-file-catalog.mjs | 56 +++++++++++++++++++ src/host-admin-server.mjs | 6 +- src/host-file-catalog.mjs | 88 ++++++++++++++++++------------ 5 files changed, 148 insertions(+), 40 deletions(-) diff --git a/docs/host-file-catalog.md b/docs/host-file-catalog.md index c7e2a0e..52bcaa0 100644 --- a/docs/host-file-catalog.md +++ b/docs/host-file-catalog.md @@ -50,7 +50,8 @@ All output is JSON. Search matches the literal query against metadata paths, inc Import `HostFileCatalog` from `src/host-file-catalog.mjs` and construct with `{databasePath,policy}`. Methods: -- `await scan({maxEntries=10000,budgetMs=2000,restart=false})`: bounded queue work, returning `processed`, `state` and status fields. +- `await scan({maxEntries=10000,budgetMs=2000,restart=false,includeStatus=true})`: bounded queue work, returning `processed`, `state` and status fields. +- `scanState()`: constant-size scan state and generation, without queue counts or inventory aggregates. Set `includeStatus:false` on `scan()` to return only `processed`, `state` and `scan`; the default still returns full status. - `status()`: `scan.state` (`idle`, `running`, `complete`), generation, pending work, counts by domain/content state, coverage errors/exclusions and freshness. - `search({query='',domain=null,limit=50,offset=0})` and `suggestions({limit=50})`. - `close()`: close handles/database and release the native SQLite connection lock. @@ -61,7 +62,7 @@ A completed catalog remains complete on subsequent `scan()` calls. Start another The database parent must be dedicated, owned by the current user, and private (0700); one missing final directory can be created. The database must be an owned 0600 regular file with one link. The catalog uses SQLite rollback journaling inside this directory. `locking_mode=EXCLUSIVE` and an initial exclusive transaction retain a native SQLite lock for the connection lifetime, with a bounded busy timeout. Only one catalog connection can own the database; this includes status/search CLI instances. Do not run a standalone CLI against the same database while an administrative server owns it; use that server's read interface. -The operating system releases the native lock when the process exits, including SIGKILL. No PID sentinel or manual unlink is needed. Reopen the existing database to resume its persisted queue. SQLite transactions preserve committed inventory and queue progress across crashes; the test suite verifies a killed child process followed by immediate reconnect/resume. +The operating system releases the native lock when the process exits, including SIGKILL. No PID sentinel or manual unlink is needed. Reopen the existing database to resume its persisted queue. Each bounded scan call commits inventory, coverage, text and queue cursors in one SQLite transaction. A failure rolls back the batch and closes directory iterators so the next call reopens them at committed offsets. A crash can repeat the last uncommitted batch; previously committed progress remains durable. The test suite verifies rollback and a killed child during an open batch followed by immediate reconnect/resume. The background service requests minimal scan results and computes full status only for an explicit status request. Traversal uses descriptor-anchored directory components and `O_NOFOLLOW`, including ancestors. Symlinks and special files are metadata-only, never content-read or traversed. Regular files with multiple hardlinks are `metadata-only-hardlink` and never content-read or proposed for reorganization, preventing an innocuous hardlink alias from bypassing protected-path rules. Link count is included in the stat signature and rechecked before and after reads. Source path strings never become shell commands. Permission errors and inaccessible paths are coverage errors. Credentials paths are rejected before content reads; allowlisted regular files are read through a descriptor with `O_NOFOLLOW` and `O_NONBLOCK`, a hard byte bound, and stat identity/change checks. Text requires strict UTF-8 without NUL. Recognizable private keys, common tokens and credential assignments are refused without storing their content. Pattern detection cannot recognize every possible unlabelled secret: use narrow roots/extensions, path exclusions, and metadata-only policy for confidential trees. diff --git a/scripts/test-host-admin-server.mjs b/scripts/test-host-admin-server.mjs index 03fafbf..bdb75bd 100644 --- a/scripts/test-host-admin-server.mjs +++ b/scripts/test-host-admin-server.mjs @@ -3,7 +3,7 @@ import assert from 'node:assert/strict'; import {mkdtemp,writeFile,mkdir,symlink,rm} from 'node:fs/promises'; import {tmpdir} from 'node:os'; import {join} from 'node:path'; -import {createHostAdminServer,normalizeHealth,normalizeQuality,readPrivateFile,validateHostAdminConfig,runHealthProbe} from '../src/host-admin-server.mjs'; +import {createHostAdminServer,normalizeHealth,normalizeQuality,readPrivateFile,validateHostAdminConfig,runHealthProbe,startHostAdmin} from '../src/host-admin-server.mjs'; const token = 'x'.repeat(48); test('every admin route refuses unauthenticated calls and writes; query input cannot select upstream resources',async () => { @@ -54,3 +54,34 @@ test('probe degraded exit status is preserved and malformed output stays unknown assert.equal(value.overall,'degraded'); assert.equal(await runHealthProbe({command:process.execPath,args:['-e',`process.stdout.write('invalid')`],timeoutMs:1000}),null); }); + + +test('background scan ticks request minimal results without aggregate status',async ()=>{ + const {HostFileCatalog}=await import('../src/host-file-catalog.mjs'); + const root=await mkdtemp(join(tmpdir(),'amf-admin-tick-')); + await mkdir(join(root,'private'),{mode:0o700});await mkdir(join(root,'source')); + await writeFile(join(root,'source','one.txt'),'example'); + const privateRoot=join(root,'private'),policyPath=join(privateRoot,'policy'),tokenFile=join(privateRoot,'token'); + await writeFile(policyPath,JSON.stringify({schema:'amf.host-file-catalog-policy/v1',hostId:'test-host',roots:[join(root,'source')],excludePrefixes:[],metadataOnlyPrefixes:[],domains:[],content:{enabled:false,maxBytes:1024,extensions:[]}}),{mode:0o600}); + await writeFile(tokenFile,token,{mode:0o600}); + // Reserve an ephemeral loopback port, then use the production config validator. + const {createServer}=await import('node:net');const reservation=createServer(); + await new Promise(resolve=>reservation.listen(0,'127.0.0.1',resolve));const port=reservation.address().port; + await new Promise(resolve=>reservation.close(resolve)); + const originalStatus=HostFileCatalog.prototype.status,originalScan=HostFileCatalog.prototype.scan; + let aggregates=0;const calls=[]; + HostFileCatalog.prototype.status=function(){aggregates++;return originalStatus.call(this);}; + HostFileCatalog.prototype.scan=function(options){calls.push(options);return originalScan.call(this,options);}; + let service; + try { + service=await startHostAdmin({schema:'amf.host-admin-config/v1',bind:'127.0.0.1',port,policyPath,tokenFile,databasePath:join(privateRoot,'catalog.sqlite'),scanIntervalMs:100,scanMaxEntries:1}); + const deadline=Date.now()+5000; + while((calls.length<4) && Date.now()setTimeout(resolve,25)); + assert.ok(calls.length>=4);assert.ok(calls.every(options=>options.includeStatus===false));assert.equal(aggregates,0); + const report=await (await fetch(`http://127.0.0.1:${port}/v1/status`,{headers:{Authorization:`Bearer ${token}`}})).json(); + assert.equal(aggregates,1);assert.equal(report.catalog.scan.state,'complete'); + } finally { + await service?.close();HostFileCatalog.prototype.status=originalStatus;HostFileCatalog.prototype.scan=originalScan; + await rm(root,{recursive:true,force:true}); + } +}); diff --git a/scripts/test-host-file-catalog.mjs b/scripts/test-host-file-catalog.mjs index bc890f9..6ef9af5 100644 --- a/scripts/test-host-file-catalog.mjs +++ b/scripts/test-host-file-catalog.mjs @@ -205,3 +205,59 @@ test('hardlinked innocuous aliases of protected files remain metadata-only witho assert.equal(reads,0);assert.equal(c.search({query:alias}).items[0].contentState,'metadata-only-hardlink'); assert.equal(c.search({query:'unlabelled'}).total,0);assert.deepEqual(c.suggestions(),[]); }); + +test('bounded calls commit once and minimal results avoid aggregate status',async t=>{ + const f=fixture(t);for(let i=0;i<20;i++) fs.writeFileSync(path.join(f.source,`${i}.txt`),'batchedword'); + const c=new HostFileCatalog(f);t.after(()=>c.close()); + const exec=c.db.exec.bind(c.db),transactions=[]; + c.db.exec=sql=>{transactions.push(sql);return exec(sql);}; + const status=c.status.bind(c);c.status=()=>{throw new Error('Aggregate status must be lazy');}; + const result=await c.scan({maxEntries:15,budgetMs:1000,includeStatus:false}); + assert.equal(result.processed,15);assert.deepEqual(Object.keys(result).sort(),['processed','scan','state']); + assert.equal(result.state,'running');assert.equal(c.scanState().generation,1); + assert.deepEqual(transactions,['BEGIN IMMEDIATE','COMMIT']); + await assert.rejects(c.scan({includeStatus:'false'}),/Invalid/); + c.status=status;await finish(c);assert.equal(c.search({query:'batchedword'}).total,20); + assert.ok((await c.scan()).counts);assert.equal((await c.scan({includeStatus:false})).state,'complete'); +}); + +test('failed batch rolls back queue cursor and resets advanced directory handles',async t=>{ + const f=fixture(t);for(let i=0;i<8;i++) fs.writeFileSync(path.join(f.source,`${i}.txt`),'rollbackword'); + const c=new HostFileCatalog(f);t.after(()=>c.close());await c.scan({maxEntries:2}); + const queue=()=>c.db.prepare('SELECT * FROM queue ORDER BY id').all(); + const before=queue(),exec=c.db.exec.bind(c.db); + c.db.exec=sql=>{if(sql==='COMMIT') throw new Error('Injected commit failure');return exec(sql);}; + await assert.rejects(c.scan({maxEntries:4}),/Injected commit failure/); + assert.deepEqual(queue(),before);assert.equal(c.handles.size,0);assert.equal(c.scanning,false); + c.db.exec=exec;await finish(c);assert.equal(c.search({query:'rollbackword'}).total,8); +}); + +test('failed generation rolls back metadata, text, coverage and state together',async t=>{ + const f=fixture(t),file=path.join(f.source,'note.txt');fs.writeFileSync(file,'originalword'); + const c=new HostFileCatalog(f);t.after(()=>c.close());await finish(c); + const before=c.status(),rows=()=>c.db.prepare('SELECT * FROM files ORDER BY path').all(),files=rows(); + fs.writeFileSync(file,'replacementword');fs.symlinkSync(file,path.join(f.source,'link.txt')); + const exec=c.db.exec.bind(c.db); + c.db.exec=sql=>{if(sql==='COMMIT') throw new Error('Injected commit failure');return exec(sql);}; + await assert.rejects(c.scan({maxEntries:100,budgetMs:1000,restart:true}),/Injected commit failure/); + assert.deepEqual(c.status(),before);assert.deepEqual(rows(),files); + assert.equal(c.search({query:'originalword'}).total,1);assert.equal(c.search({query:'replacementword'}).total,0); + c.db.exec=exec;await finish(c,{restart:true});assert.equal(c.search({query:'replacementword'}).total,1); +}); + +test('SIGKILL during an open batch preserves committed cursor and rolls back text', {timeout:30000},async t=>{ + const f=fixture(t);for(let i=0;i<8;i++) fs.writeFileSync(path.join(f.source,`${i}.txt`),'crashbatchword'); + const code=`import fs from 'node:fs';import {HostFileCatalog} from './src/host-file-catalog.mjs'; + const c=new HostFileCatalog(JSON.parse(process.argv[1]));await c.scan({maxEntries:2}); + const inspect=c.inspect.bind(c);c.inspect=p=>{inspect(p);fs.writeSync(1,'uncommitted\\n');Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0);}; + await c.scan({maxEntries:100,budgetMs:10000});`; + const child=spawn(process.execPath,['--input-type=module','-e',code,JSON.stringify({databasePath:f.databasePath,policy:f.policy})],{stdio:['ignore','pipe','pipe']}); + t.after(()=>{if(child.exitCode===null)child.kill('SIGKILL');}); + await new Promise((resolve,reject)=>{child.stdout.once('data',resolve);child.once('error',reject);child.once('exit',()=>reject(new Error('Child exited before uncommitted work')));}); + const killed=new Promise(resolve=>child.once('exit',resolve));child.kill('SIGKILL');await killed; + const c=new HostFileCatalog(f);t.after(()=>c.close()); + assert.equal(c.scanState().state,'running');assert.equal(c.search({query:'crashbatchword'}).total,0); + assert.equal(c.db.prepare("SELECT cursor FROM queue WHERE kind='directory'").get().cursor,1); + await finish(c);assert.equal(c.search({query:'crashbatchword'}).total,8); + assert.equal(c.db.prepare('SELECT count(*) AS n FROM texts').get().n,8); +}); diff --git a/src/host-admin-server.mjs b/src/host-admin-server.mjs index 0edc25b..a0d3bbf 100644 --- a/src/host-admin-server.mjs +++ b/src/host-admin-server.mjs @@ -146,10 +146,10 @@ export async function startHostAdmin(rawConfig) { let stopped = false,scanning = null,lastComplete = 0; const tick = async () => { if (stopped || scanning) return; - const state = catalog.status().scan?.state; + const state = catalog.scanState().state; if (state === 'complete' && Date.now() - lastComplete < config.rescanIntervalMs) return; - scanning = catalog.scan({maxEntries:config.scanMaxEntries,budgetMs:config.scanBudgetMs,restart:state === 'complete'}); - try { await scanning; api.setScannerError(null); if (catalog.status().scan?.state === 'complete') lastComplete = Date.now(); } + scanning = catalog.scan({maxEntries:config.scanMaxEntries,budgetMs:config.scanBudgetMs,restart:state === 'complete',includeStatus:false}); + try { const result = await scanning; api.setScannerError(null); if (result.state === 'complete') lastComplete = Date.now(); } catch {api.setScannerError('scan_failed');} finally {scanning = null;} }; diff --git a/src/host-file-catalog.mjs b/src/host-file-catalog.mjs index c4d45a3..2a2cae2 100644 --- a/src/host-file-catalog.mjs +++ b/src/host-file-catalog.mjs @@ -146,36 +146,55 @@ export class HostFileCatalog { if (task.cursor >= this.maxDirectoryEntries) { this.record(task.path,'incomplete','directory-entry-limit'); handle.directory.closeSync(); this.handles.delete(task.id); return true; } const entry = handle.directory.readSync(); if (!entry) { handle.directory.closeSync(); this.handles.delete(task.id); return true; } - this.db.prepare("INSERT INTO queue(path,kind) VALUES(?,'entry')").run(path.join(task.path,entry.name)); + const child = this.db.prepare("INSERT INTO queue(path,kind) VALUES(?,'entry')").run(path.join(task.path,entry.name)); this.db.prepare('UPDATE queue SET cursor=cursor+1 WHERE id=?').run(task.id); - return false; + return {done:false,childId:child.lastInsertRowid}; } - async scan({ maxEntries=10000, budgetMs=2000, restart=false }={}) { + closeHandles() { + for (const active of this.handles.values()) { try { active.directory.closeSync(); } catch {} } + this.handles.clear(); + } + scanState() { + const state = this.db.prepare('SELECT generation,phase,started,completed FROM state').get(); + return {...state,state:state.phase === 'scanning' ? 'running' : state.phase}; + } + async scan({ maxEntries=10000, budgetMs=2000, restart=false, includeStatus=true }={}) { int(maxEntries,1,100000,'maxEntries'); int(budgetMs,1,60000,'budgetMs'); - if (typeof restart !== 'boolean') throw new Error('Invalid restart'); + if (typeof restart !== 'boolean' || typeof includeStatus !== 'boolean') throw new Error('Invalid scan options'); if (this.scanning) throw new Error('Scan already running'); this.scanning = true; + const result = processed => { + const status = includeStatus ? this.status() : {scan:this.scanState()}; + return {processed,state:status.scan.state,...status}; + }; try { - let state = this.db.prepare('SELECT * FROM state').get(); - if (state.phase === 'complete' && !restart) return { processed:0, state:'complete', ...this.status() }; - if (state.phase !== 'scanning') { - this.db.exec('BEGIN IMMEDIATE'); - try { + const state = this.scanState(); + if (state.state === 'complete' && !restart) return result(0); + const start = Date.now();let processed = 0,nextTaskId = null; + // One durable transaction per bounded batch: cursor, child queue, + // inventory and FTS either commit together or all remain at the last tick. + this.db.exec('BEGIN IMMEDIATE'); + try { + if (state.phase !== 'scanning') { + this.closeHandles(); this.db.prepare("UPDATE state SET generation=generation+1,phase='scanning',completed=NULL,started=?,policy_hash=? WHERE id=1").run(now(),this.policyHash); for (const root of [...new Set(this.policy.roots)].filter(r => !this.policy.roots.some(other => other !== r && within(r,other)))) this.db.prepare("INSERT INTO queue(path,kind) VALUES(?,'entry')").run(root); - this.db.exec('COMMIT'); - } catch(error) { this.db.exec('ROLLBACK'); throw error; } - } - this.generation = this.db.prepare('SELECT generation FROM state').get().generation; - const start = Date.now(); let processed = 0; - while (processed < maxEntries && Date.now()-start < budgetMs) { - const task = this.db.prepare('SELECT * FROM queue ORDER BY id LIMIT 1').get(); - if (!task) break; - this.db.exec('BEGIN IMMEDIATE'); - try { + } + this.generation = this.scanState().generation; + while (processed < maxEntries && Date.now()-start < budgetMs) { + const task = nextTaskId === null ? this.db.prepare('SELECT * FROM queue ORDER BY id LIMIT 1').get() : this.db.prepare('SELECT * FROM queue WHERE id=?').get(nextTaskId); + nextTaskId = null; + if (!task) break; let done = true; - try { if (task.kind === 'directory') done = this.directory(task); else this.inspect(task.path); } - catch(error) { + try { + if (task.kind === 'directory') { + const progress = this.directory(task); + done = typeof progress === 'object' ? progress.done : progress; + // Inspect each newly discovered child before continuing its + // directory, so a large directory cannot monopolize inventory. + if (typeof progress === 'object') nextTaskId = progress.childId; + } else this.inspect(task.path); + } catch(error) { this.record(task.path,'error',error.code ?? 'inspection-failed'); this.db.prepare('DELETE FROM texts WHERE path=?').run(task.path); this.db.prepare("UPDATE files SET content_state='unavailable' WHERE path=?").run(task.path); @@ -183,23 +202,24 @@ export class HostFileCatalog { if (active) { try { active.directory.closeSync(); } catch {} this.handles.delete(task.id); } } if (done) this.db.prepare('DELETE FROM queue WHERE id=?').run(task.id); - this.db.exec('COMMIT'); - } catch(error) { - this.db.exec('ROLLBACK'); - for (const active of this.handles.values()) { try { active.directory.closeSync(); } catch {} } - this.handles.clear(); - throw error; + processed++; } - processed++; + if (!this.db.prepare('SELECT 1 FROM queue LIMIT 1').get()) this.db.prepare("UPDATE state SET phase='complete',completed=? WHERE id=1").run(now()); + this.db.exec('COMMIT'); + } catch(error) { + try { this.db.exec('ROLLBACK'); } finally { + // Iterators can have advanced beyond the rolled-back durable cursor. + // Reopen from that cursor next time; never reuse their in-memory offset. + this.closeHandles(); + } + throw error; } - if (!this.db.prepare('SELECT 1 FROM queue LIMIT 1').get()) this.db.prepare("UPDATE state SET phase='complete',completed=? WHERE id=1").run(now()); - const status = this.status(); - return { processed, state:status.scan.state, ...status }; + return result(processed); } finally { this.scanning = false; } } status() { - const state = this.db.prepare('SELECT generation,phase,started,completed FROM state').get(); - return { hostId:this.policy.hostId, scan:{...state,state:state.phase === 'scanning' ? 'running' : state.phase}, pending:this.db.prepare('SELECT count(*) AS n FROM queue').get().n, + const state = this.scanState(); + return { hostId:this.policy.hostId, scan:state, pending:this.db.prepare('SELECT count(*) AS n FROM queue').get().n, counts:{ total:this.db.prepare('SELECT count(*) AS n FROM files').get().n, byDomain:this.db.prepare('SELECT domain,count(*) AS count FROM files GROUP BY domain').all(), byContentState:this.db.prepare('SELECT content_state AS state,count(*) AS count FROM files GROUP BY content_state').all() }, coverage:{ roots:this.policy.roots, configuredExclusions:this.policy.excludePrefixes, metadataOnlyPrefixes:this.policy.metadataOnlyPrefixes, metadataOnlyDirectoryNames:this.policy.metadataOnlyDirectoryNames ?? [], summary:this.db.prepare('SELECT kind,count(*) AS count FROM coverage WHERE generation=? GROUP BY kind').all(state.generation), recent:this.db.prepare('SELECT path,kind,reason,observed_at FROM coverage WHERE generation=? ORDER BY id DESC LIMIT 100').all(state.generation) }, freshness:{ unverified:this.db.prepare('SELECT count(*) AS n FROM files WHERE generation<>?').get(state.generation).n, semantics:'Last observed metadata; unseen files are retained as unverified, never inferred deleted. Concurrent directory changes may require another generation.' } }; @@ -250,7 +270,7 @@ export class HostFileCatalog { close() { if (this.closed) return; this.closed = true; - for (const h of this.handles?.values() ?? []) { try { h.directory.closeSync(); } catch {} } + this.closeHandles(); this.db?.close(); }