Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions docs/host-file-catalog.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ All output is JSON. Search matches the literal query against metadata paths, inc

Import `HostFileCatalog` from `src/host-file-catalog.mjs` and construct with `{databasePath,policy}`. Methods:

- `await scan({maxEntries=10000,budgetMs=2000,restart=false})`: bounded queue work, returning `processed`, `state` and status fields.
- `await scan({maxEntries=10000,budgetMs=2000,restart=false,includeStatus=true})`: bounded queue work, returning `processed`, `state` and status fields.
- `scanState()`: constant-size scan state and generation, without queue counts or inventory aggregates. Set `includeStatus:false` on `scan()` to return only `processed`, `state` and `scan`; the default still returns full status.
- `status()`: `scan.state` (`idle`, `running`, `complete`), generation, pending work, counts by domain/content state, coverage errors/exclusions and freshness.
- `search({query='',domain=null,limit=50,offset=0})` and `suggestions({limit=50})`.
- `close()`: close handles/database and release the native SQLite connection lock.
Expand All @@ -61,7 +62,7 @@ A completed catalog remains complete on subsequent `scan()` calls. Start another

The database parent must be dedicated, owned by the current user, and private (0700); one missing final directory can be created. The database must be an owned 0600 regular file with one link. The catalog uses SQLite rollback journaling inside this directory. `locking_mode=EXCLUSIVE` and an initial exclusive transaction retain a native SQLite lock for the connection lifetime, with a bounded busy timeout. Only one catalog connection can own the database; this includes status/search CLI instances. Do not run a standalone CLI against the same database while an administrative server owns it; use that server's read interface.

The operating system releases the native lock when the process exits, including SIGKILL. No PID sentinel or manual unlink is needed. Reopen the existing database to resume its persisted queue. SQLite transactions preserve committed inventory and queue progress across crashes; the test suite verifies a killed child process followed by immediate reconnect/resume.
The operating system releases the native lock when the process exits, including SIGKILL. No PID sentinel or manual unlink is needed. Reopen the existing database to resume its persisted queue. Each bounded scan call commits inventory, coverage, text and queue cursors in one SQLite transaction. A failure rolls back the batch and closes directory iterators so the next call reopens them at committed offsets. A crash can repeat the last uncommitted batch; previously committed progress remains durable. The test suite verifies rollback and a killed child during an open batch followed by immediate reconnect/resume. The background service requests minimal scan results and computes full status only for an explicit status request.

Traversal uses descriptor-anchored directory components and `O_NOFOLLOW`, including ancestors. Symlinks and special files are metadata-only, never content-read or traversed. Regular files with multiple hardlinks are `metadata-only-hardlink` and never content-read or proposed for reorganization, preventing an innocuous hardlink alias from bypassing protected-path rules. Link count is included in the stat signature and rechecked before and after reads. Source path strings never become shell commands. Permission errors and inaccessible paths are coverage errors. Credentials paths are rejected before content reads; allowlisted regular files are read through a descriptor with `O_NOFOLLOW` and `O_NONBLOCK`, a hard byte bound, and stat identity/change checks. Text requires strict UTF-8 without NUL. Recognizable private keys, common tokens and credential assignments are refused without storing their content. Pattern detection cannot recognize every possible unlabelled secret: use narrow roots/extensions, path exclusions, and metadata-only policy for confidential trees.

Expand Down
33 changes: 32 additions & 1 deletion scripts/test-host-admin-server.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import assert from 'node:assert/strict';
import {mkdtemp,writeFile,mkdir,symlink,rm} from 'node:fs/promises';
import {tmpdir} from 'node:os';
import {join} from 'node:path';
import {createHostAdminServer,normalizeHealth,normalizeQuality,readPrivateFile,validateHostAdminConfig,runHealthProbe} from '../src/host-admin-server.mjs';
import {createHostAdminServer,normalizeHealth,normalizeQuality,readPrivateFile,validateHostAdminConfig,runHealthProbe,startHostAdmin} from '../src/host-admin-server.mjs';

const token = 'x'.repeat(48);
test('every admin route refuses unauthenticated calls and writes; query input cannot select upstream resources',async () => {
Expand Down Expand Up @@ -54,3 +54,34 @@ test('probe degraded exit status is preserved and malformed output stays unknown
assert.equal(value.overall,'degraded');
assert.equal(await runHealthProbe({command:process.execPath,args:['-e',`process.stdout.write('invalid')`],timeoutMs:1000}),null);
});


test('background scan ticks request minimal results without aggregate status',async ()=>{
const {HostFileCatalog}=await import('../src/host-file-catalog.mjs');
const root=await mkdtemp(join(tmpdir(),'amf-admin-tick-'));
await mkdir(join(root,'private'),{mode:0o700});await mkdir(join(root,'source'));
await writeFile(join(root,'source','one.txt'),'example');
const privateRoot=join(root,'private'),policyPath=join(privateRoot,'policy'),tokenFile=join(privateRoot,'token');
await writeFile(policyPath,JSON.stringify({schema:'amf.host-file-catalog-policy/v1',hostId:'test-host',roots:[join(root,'source')],excludePrefixes:[],metadataOnlyPrefixes:[],domains:[],content:{enabled:false,maxBytes:1024,extensions:[]}}),{mode:0o600});
await writeFile(tokenFile,token,{mode:0o600});
// Reserve an ephemeral loopback port, then use the production config validator.
const {createServer}=await import('node:net');const reservation=createServer();
await new Promise(resolve=>reservation.listen(0,'127.0.0.1',resolve));const port=reservation.address().port;
await new Promise(resolve=>reservation.close(resolve));
const originalStatus=HostFileCatalog.prototype.status,originalScan=HostFileCatalog.prototype.scan;
let aggregates=0;const calls=[];
HostFileCatalog.prototype.status=function(){aggregates++;return originalStatus.call(this);};
HostFileCatalog.prototype.scan=function(options){calls.push(options);return originalScan.call(this,options);};
let service;
try {
service=await startHostAdmin({schema:'amf.host-admin-config/v1',bind:'127.0.0.1',port,policyPath,tokenFile,databasePath:join(privateRoot,'catalog.sqlite'),scanIntervalMs:100,scanMaxEntries:1});
const deadline=Date.now()+5000;
while((calls.length<4) && Date.now()<deadline) await new Promise(resolve=>setTimeout(resolve,25));
assert.ok(calls.length>=4);assert.ok(calls.every(options=>options.includeStatus===false));assert.equal(aggregates,0);
const report=await (await fetch(`http://127.0.0.1:${port}/v1/status`,{headers:{Authorization:`Bearer ${token}`}})).json();
assert.equal(aggregates,1);assert.equal(report.catalog.scan.state,'complete');
} finally {
await service?.close();HostFileCatalog.prototype.status=originalStatus;HostFileCatalog.prototype.scan=originalScan;
await rm(root,{recursive:true,force:true});
}
});
56 changes: 56 additions & 0 deletions scripts/test-host-file-catalog.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -205,3 +205,59 @@ test('hardlinked innocuous aliases of protected files remain metadata-only witho
assert.equal(reads,0);assert.equal(c.search({query:alias}).items[0].contentState,'metadata-only-hardlink');
assert.equal(c.search({query:'unlabelled'}).total,0);assert.deepEqual(c.suggestions(),[]);
});

test('bounded calls commit once and minimal results avoid aggregate status',async t=>{
const f=fixture(t);for(let i=0;i<20;i++) fs.writeFileSync(path.join(f.source,`${i}.txt`),'batchedword');
const c=new HostFileCatalog(f);t.after(()=>c.close());
const exec=c.db.exec.bind(c.db),transactions=[];
c.db.exec=sql=>{transactions.push(sql);return exec(sql);};
const status=c.status.bind(c);c.status=()=>{throw new Error('Aggregate status must be lazy');};
const result=await c.scan({maxEntries:15,budgetMs:1000,includeStatus:false});
assert.equal(result.processed,15);assert.deepEqual(Object.keys(result).sort(),['processed','scan','state']);
assert.equal(result.state,'running');assert.equal(c.scanState().generation,1);
assert.deepEqual(transactions,['BEGIN IMMEDIATE','COMMIT']);
await assert.rejects(c.scan({includeStatus:'false'}),/Invalid/);
c.status=status;await finish(c);assert.equal(c.search({query:'batchedword'}).total,20);
assert.ok((await c.scan()).counts);assert.equal((await c.scan({includeStatus:false})).state,'complete');
});

test('failed batch rolls back queue cursor and resets advanced directory handles',async t=>{
const f=fixture(t);for(let i=0;i<8;i++) fs.writeFileSync(path.join(f.source,`${i}.txt`),'rollbackword');
const c=new HostFileCatalog(f);t.after(()=>c.close());await c.scan({maxEntries:2});
const queue=()=>c.db.prepare('SELECT * FROM queue ORDER BY id').all();
const before=queue(),exec=c.db.exec.bind(c.db);
c.db.exec=sql=>{if(sql==='COMMIT') throw new Error('Injected commit failure');return exec(sql);};
await assert.rejects(c.scan({maxEntries:4}),/Injected commit failure/);
assert.deepEqual(queue(),before);assert.equal(c.handles.size,0);assert.equal(c.scanning,false);
c.db.exec=exec;await finish(c);assert.equal(c.search({query:'rollbackword'}).total,8);
});

test('failed generation rolls back metadata, text, coverage and state together',async t=>{
const f=fixture(t),file=path.join(f.source,'note.txt');fs.writeFileSync(file,'originalword');
const c=new HostFileCatalog(f);t.after(()=>c.close());await finish(c);
const before=c.status(),rows=()=>c.db.prepare('SELECT * FROM files ORDER BY path').all(),files=rows();
fs.writeFileSync(file,'replacementword');fs.symlinkSync(file,path.join(f.source,'link.txt'));
const exec=c.db.exec.bind(c.db);
c.db.exec=sql=>{if(sql==='COMMIT') throw new Error('Injected commit failure');return exec(sql);};
await assert.rejects(c.scan({maxEntries:100,budgetMs:1000,restart:true}),/Injected commit failure/);
assert.deepEqual(c.status(),before);assert.deepEqual(rows(),files);
assert.equal(c.search({query:'originalword'}).total,1);assert.equal(c.search({query:'replacementword'}).total,0);
c.db.exec=exec;await finish(c,{restart:true});assert.equal(c.search({query:'replacementword'}).total,1);
});

test('SIGKILL during an open batch preserves committed cursor and rolls back text', {timeout:30000},async t=>{
const f=fixture(t);for(let i=0;i<8;i++) fs.writeFileSync(path.join(f.source,`${i}.txt`),'crashbatchword');
const code=`import fs from 'node:fs';import {HostFileCatalog} from './src/host-file-catalog.mjs';
const c=new HostFileCatalog(JSON.parse(process.argv[1]));await c.scan({maxEntries:2});
const inspect=c.inspect.bind(c);c.inspect=p=>{inspect(p);fs.writeSync(1,'uncommitted\\n');Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0);};
await c.scan({maxEntries:100,budgetMs:10000});`;
const child=spawn(process.execPath,['--input-type=module','-e',code,JSON.stringify({databasePath:f.databasePath,policy:f.policy})],{stdio:['ignore','pipe','pipe']});
t.after(()=>{if(child.exitCode===null)child.kill('SIGKILL');});
await new Promise((resolve,reject)=>{child.stdout.once('data',resolve);child.once('error',reject);child.once('exit',()=>reject(new Error('Child exited before uncommitted work')));});
const killed=new Promise(resolve=>child.once('exit',resolve));child.kill('SIGKILL');await killed;
const c=new HostFileCatalog(f);t.after(()=>c.close());
assert.equal(c.scanState().state,'running');assert.equal(c.search({query:'crashbatchword'}).total,0);
assert.equal(c.db.prepare("SELECT cursor FROM queue WHERE kind='directory'").get().cursor,1);
await finish(c);assert.equal(c.search({query:'crashbatchword'}).total,8);
assert.equal(c.db.prepare('SELECT count(*) AS n FROM texts').get().n,8);
});
6 changes: 3 additions & 3 deletions src/host-admin-server.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -146,10 +146,10 @@ export async function startHostAdmin(rawConfig) {
let stopped = false,scanning = null,lastComplete = 0;
const tick = async () => {
if (stopped || scanning) return;
const state = catalog.status().scan?.state;
const state = catalog.scanState().state;
if (state === 'complete' && Date.now() - lastComplete < config.rescanIntervalMs) return;
scanning = catalog.scan({maxEntries:config.scanMaxEntries,budgetMs:config.scanBudgetMs,restart:state === 'complete'});
try { await scanning; api.setScannerError(null); if (catalog.status().scan?.state === 'complete') lastComplete = Date.now(); }
scanning = catalog.scan({maxEntries:config.scanMaxEntries,budgetMs:config.scanBudgetMs,restart:state === 'complete',includeStatus:false});
try { const result = await scanning; api.setScannerError(null); if (result.state === 'complete') lastComplete = Date.now(); }
catch {api.setScannerError('scan_failed');}
finally {scanning = null;}
};
Expand Down
Loading
Loading