From 4714d1fd427003b9d097ddd388cfe5b851d55d5c Mon Sep 17 00:00:00 2001 From: Terry <186405119+tlq5l@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:02:38 +0700 Subject: [PATCH 1/8] Respect another run's file lock in --reinvestigate and direct mode `process --reinvestigate` and direct mode (`--diff`, `--files`, `--files-from`) claimed every selected file, including a file that another run had locked and was still investigating. Each run writes the whole FileRecord back from its own snapshot, so the later write dropped the other run's findings and analysis history. The claim step now skips a `processing` file whose lock isn't reclaimable, which is the rule the default path already applies. Analyzed files, abandoned locks, and the resumed run's own files are still claimed. --- .../src/__tests__/process-revalidate.test.ts | 62 +++++++++++++++++++ packages/processor/src/index.ts | 4 +- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/packages/processor/src/__tests__/process-revalidate.test.ts b/packages/processor/src/__tests__/process-revalidate.test.ts index 7c5c3c90..1d0880a4 100644 --- a/packages/processor/src/__tests__/process-revalidate.test.ts +++ b/packages/processor/src/__tests__/process-revalidate.test.ts @@ -458,6 +458,68 @@ describe("processor with stub agent", () => { expect(after.lockedByRunId).toBe(liveRunId); }); + it.each([ + ["--reinvestigate", { reinvestigate: true }], + ["a file list", { filePaths: ["live.ts", "dead.ts", "mine.ts", "done.ts"] }], + ])("process() with %s does NOT take a file a still-running other run holds", async (_mode, forceParams) => { + // Both modes select files whatever their status, so they can re-claim + // analyzed files. Only the other run's non-reclaimable lock must survive. + const fx = setupProject({ files: ["live.ts", "dead.ts", "mine.ts", "done.ts"] }); + const liveRunId = "20260101000000-liveforceaaaaaaa"; + const deadRunId = "20260101000000-deadforceaaaaaaa"; + const ownRunId = "20260101000000-ownforceaaaaaaaa"; + + fs.mkdirSync(path.join(fx.dataRoot, fx.projectId, "runs"), { recursive: true }); + const lock = (filePath: string, runId: string, pid: number) => { + const rec = pendingRecord(fx.projectId, filePath); + rec.status = "processing"; + rec.lockedByRunId = runId; + rec.lockedAt = new Date().toISOString(); + fx.writeRecord(rec); + fs.writeFileSync( + path.join(fx.dataRoot, fx.projectId, "runs", `${runId}.json`), + JSON.stringify({ + runId, + projectId: fx.projectId, + rootPath: fx.targetRoot, + createdAt: new Date().toISOString(), + type: "process", + phase: "running", + pid, + hostname: os.hostname(), + stats: {}, + }), + ); + }; + lock("live.ts", liveRunId, process.pid); + const liveBefore = fx.readRecord("live.ts"); + lock("dead.ts", deadRunId, 0x7fffffff); + lock("mine.ts", ownRunId, process.pid); + const done = pendingRecord(fx.projectId, "done.ts"); + done.status = "analyzed"; + fx.writeRecord(done); + + const stub = new StubAgent(); + setLoadedConfig( + defineConfig({ + projects: [{ id: fx.projectId, root: fx.targetRoot }], + plugins: [{ name: "stub", agents: [stub] }], + }), + ); + + await processProject({ + projectId: fx.projectId, + agentType: "stub", + concurrency: 1, + runId: ownRunId, + ...forceParams, + }); + + const investigated = stub.calls.investigateCalls.flatMap((c) => c.batch.map((r) => r.filePath)); + expect(investigated.sort()).toEqual(["dead.ts", "done.ts", "mine.ts"]); + expect(fx.readRecord("live.ts")).toEqual(liveBefore); + }); + it("process() captures refusals from the agent into AnalysisEntry", async () => { const fx = setupProject({ files: ["app.ts"] }); fx.writeRecord(pendingRecord(fx.projectId, "app.ts")); diff --git a/packages/processor/src/index.ts b/packages/processor/src/index.ts index 09bd3923..4c144b10 100644 --- a/packages/processor/src/index.ts +++ b/packages/processor/src/index.ts @@ -546,7 +546,9 @@ export async function process(params: { (current.status === "processing" && current.lockedByRunId !== runId && isReclaimableLock(current)); - if (!isOurs && !isFreelyClaimable && !inForceMode) { + // Force mode re-claims analyzed files. A file under another run's + // non-reclaimable lock stays with that run. + if (!isOurs && !isFreelyClaimable && (!inForceMode || current.status === "processing")) { continue; } From b5a275e1747473b701436435ddf56558a26b17ad Mon Sep 17 00:00:00 2001 From: Terry <186405119+tlq5l@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:12:23 +0700 Subject: [PATCH 2/8] Take cross-host locks in force mode and queue the claimed record In `--reinvestigate` and direct mode, the claim step skipped every `processing` file whose lock wasn't reclaimable. A sandbox re-run after a worker crash couldn't re-claim that worker's files, because the PID of a run on another host can't be probed. An `analyzed` record that still carried a live lock was claimed. The claim step now skips a file only when its lock isn't reclaimable and the owning run started on this host, whatever the record's status. Locks held by a run on another host are taken, as on `main`. It also queues the record it just claimed, not the startup snapshot, so findings another run wrote after the snapshot was loaded aren't overwritten. --- .../src/__tests__/process-revalidate.test.ts | 55 +++++++++++++++++++ packages/processor/src/index.ts | 29 +++++++--- 2 files changed, 75 insertions(+), 9 deletions(-) diff --git a/packages/processor/src/__tests__/process-revalidate.test.ts b/packages/processor/src/__tests__/process-revalidate.test.ts index 1d0880a4..9f6c1806 100644 --- a/packages/processor/src/__tests__/process-revalidate.test.ts +++ b/packages/processor/src/__tests__/process-revalidate.test.ts @@ -520,6 +520,61 @@ describe("processor with stub agent", () => { expect(fx.readRecord("live.ts")).toEqual(liveBefore); }); + it.each([ + [ + "takes a processing file locked from another host", + "processing", + "other-host", + "remote.ts", + true, + ], + ["leaves an analyzed file locked on this host", "analyzed", os.hostname(), "kept.ts", false], + ] as const)("process() with --reinvestigate %s", async (_name, status, hostname, filePath, taken) => { + // A lock from another host can't be PID-probed, so it is taken as + // before (sandbox re-runs). A live same-host lock holds whatever the status. + const fx = setupProject({ files: [filePath] }); + const liveRunId = "20260101000000-liveremoteaaaaaa"; + fs.mkdirSync(path.join(fx.dataRoot, fx.projectId, "runs"), { recursive: true }); + const rec = pendingRecord(fx.projectId, filePath); + rec.status = status; + rec.lockedByRunId = liveRunId; + rec.lockedAt = new Date().toISOString(); + fx.writeRecord(rec); + fs.writeFileSync( + path.join(fx.dataRoot, fx.projectId, "runs", `${liveRunId}.json`), + JSON.stringify({ + runId: liveRunId, + projectId: fx.projectId, + rootPath: fx.targetRoot, + createdAt: new Date().toISOString(), + type: "process", + phase: "running", + pid: process.pid, + hostname, + stats: {}, + }), + ); + const before = fx.readRecord(filePath); + + const stub = new StubAgent(); + setLoadedConfig( + defineConfig({ + projects: [{ id: fx.projectId, root: fx.targetRoot }], + plugins: [{ name: "stub", agents: [stub] }], + }), + ); + await processProject({ + projectId: fx.projectId, + agentType: "stub", + concurrency: 1, + reinvestigate: true, + }); + + const investigated = stub.calls.investigateCalls.flatMap((c) => c.batch.map((r) => r.filePath)); + expect(investigated).toEqual(taken ? [filePath] : []); + if (!taken) expect(fx.readRecord(filePath)).toEqual(before); + }); + it("process() captures refusals from the agent into AnalysisEntry", async () => { const fx = setupProject({ files: ["app.ts"] }); fx.writeRecord(pendingRecord(fx.projectId, "app.ts")); diff --git a/packages/processor/src/index.ts b/packages/processor/src/index.ts index 4c144b10..09693e95 100644 --- a/packages/processor/src/index.ts +++ b/packages/processor/src/index.ts @@ -391,6 +391,17 @@ export async function process(params: { return ageMs >= STALE_LOCK_MS; }; + // True when a lock held by another run can be checked (it started on + // this host) and isn't reclaimable. Force mode leaves those files alone. + const isLiveLocalLock = (r: FileRecord): boolean => { + if (isReclaimableLock(r)) return false; + try { + return readRunMeta(projectId, r.lockedByRunId!).hostname === localHostname; + } catch { + return false; + } + }; + // Load file records and pick which to process const allRecords = loadAllFileRecords(projectId); let toProcess: FileRecord[]; @@ -546,9 +557,11 @@ export async function process(params: { (current.status === "processing" && current.lockedByRunId !== runId && isReclaimableLock(current)); - // Force mode re-claims analyzed files. A file under another run's - // non-reclaimable lock stays with that run. - if (!isOurs && !isFreelyClaimable && (!inForceMode || current.status === "processing")) { + // Force mode re-claims analyzed files and files locked by a run on + // another host, whose PID we can't probe. It leaves a file alone + // only while a run on this host holds a live lock on it, whatever + // the record's status. + if (!isOurs && !isFreelyClaimable && (!inForceMode || isLiveLocalLock(current))) { continue; } @@ -557,12 +570,10 @@ export async function process(params: { current.lockedAt = lockedAt; writeFileRecord(current); - // Mutate the in-memory snapshot we'll process from so downstream code - // sees the same lock state. - record.status = current.status; - record.lockedByRunId = current.lockedByRunId; - record.lockedAt = current.lockedAt; - claimed.push(record); + // Process from the record we just claimed, not the startup snapshot: + // another run may have finished this file since the snapshot was + // loaded, and its findings and history live only in `current`. + claimed.push(current); } } finally { releaseProcessLock(); From ffae97dbf3cb86b55c8e29e8be6bfa66c1cb1a73 Mon Sep 17 00:00:00 2001 From: Terry <186405119+tlq5l@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:47:27 +0700 Subject: [PATCH 3/8] Respect live locks from any host in force mode and warn on skipped files In `--reinvestigate` and direct mode, the claim step left a file alone only when the run holding its lock started on this host. A lock held by a run on another host was taken even though it wasn't reclaimable, so two runs sharing a data directory could still investigate the same file and overwrite each other's findings and history. The claim step now leaves a `processing` or `analyzed` file alone while another run's lock on it isn't reclaimable, whatever host that run is on. A lock is reclaimable when its run is done, errored, or missing, when the run's PID is dead on this host, or when the lock is 1 hour old, as in the default path. Analyzed files without a live lock and the resumed run's own files are still claimed. Direct mode (`--diff`, `--files`, `--files-from`) now prints one warning that counts the requested files it skipped because another run holds them and names up to five. The exit code is unchanged. A sandbox re-run with `--reinvestigate` soon after a worker crashed now skips that worker's files until their locks become reclaimable, because the PID of a run on another host can't be probed. --- .../src/__tests__/process-revalidate.test.ts | 86 +++++++++++++++++-- packages/processor/src/index.ts | 27 +++--- 2 files changed, 88 insertions(+), 25 deletions(-) diff --git a/packages/processor/src/__tests__/process-revalidate.test.ts b/packages/processor/src/__tests__/process-revalidate.test.ts index 9f6c1806..67f5ada3 100644 --- a/packages/processor/src/__tests__/process-revalidate.test.ts +++ b/packages/processor/src/__tests__/process-revalidate.test.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { defineConfig, type FileRecord, setLoadedConfig } from "@deepsec/core"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { QuotaExhaustedError } from "../agents/shared.js"; import { process as processProject, revalidate } from "../index.js"; import { StubAgent } from "./stub-agent.js"; @@ -521,24 +521,42 @@ describe("processor with stub agent", () => { }); it.each([ + ["leaves a fresh lock from another host", "processing", "other-host", "running", 0, false], [ - "takes a processing file locked from another host", + "takes a lock from another host older than 1 hour", "processing", "other-host", - "remote.ts", + "running", + 61 * 60_000, true, ], - ["leaves an analyzed file locked on this host", "analyzed", os.hostname(), "kept.ts", false], - ] as const)("process() with --reinvestigate %s", async (_name, status, hostname, filePath, taken) => { - // A lock from another host can't be PID-probed, so it is taken as - // before (sandbox re-runs). A live same-host lock holds whatever the status. + [ + "takes a lock from another host whose run is done", + "processing", + "other-host", + "done", + 0, + true, + ], + [ + "leaves an analyzed file with a live lock on this host", + "analyzed", + os.hostname(), + "running", + 0, + false, + ], + ] as const)("process() with --reinvestigate %s", async (_name, status, hostname, phase, ageMs, taken) => { + // Force mode skips a file exactly when its lock isn't reclaimable, + // whatever the host and whatever the record's status. + const filePath = "locked.ts"; const fx = setupProject({ files: [filePath] }); const liveRunId = "20260101000000-liveremoteaaaaaa"; fs.mkdirSync(path.join(fx.dataRoot, fx.projectId, "runs"), { recursive: true }); const rec = pendingRecord(fx.projectId, filePath); rec.status = status; rec.lockedByRunId = liveRunId; - rec.lockedAt = new Date().toISOString(); + rec.lockedAt = new Date(Date.now() - ageMs).toISOString(); fx.writeRecord(rec); fs.writeFileSync( path.join(fx.dataRoot, fx.projectId, "runs", `${liveRunId}.json`), @@ -548,7 +566,7 @@ describe("processor with stub agent", () => { rootPath: fx.targetRoot, createdAt: new Date().toISOString(), type: "process", - phase: "running", + phase, pid: process.pid, hostname, stats: {}, @@ -575,6 +593,56 @@ describe("processor with stub agent", () => { if (!taken) expect(fx.readRecord(filePath)).toEqual(before); }); + it("process() with a file list warns about requested files another run holds", async () => { + const fx = setupProject({ files: ["held.ts", "free.ts"] }); + const otherRunId = "20260101000000-otherwarnaaaaaaa"; + fs.mkdirSync(path.join(fx.dataRoot, fx.projectId, "runs"), { recursive: true }); + const held = pendingRecord(fx.projectId, "held.ts"); + held.status = "processing"; + held.lockedByRunId = otherRunId; + held.lockedAt = new Date().toISOString(); + fx.writeRecord(held); + fx.writeRecord(pendingRecord(fx.projectId, "free.ts")); + fs.writeFileSync( + path.join(fx.dataRoot, fx.projectId, "runs", `${otherRunId}.json`), + JSON.stringify({ + runId: otherRunId, + projectId: fx.projectId, + rootPath: fx.targetRoot, + createdAt: new Date().toISOString(), + type: "process", + phase: "running", + pid: process.pid, + hostname: "other-host", + stats: {}, + }), + ); + + const stub = new StubAgent(); + setLoadedConfig( + defineConfig({ + projects: [{ id: fx.projectId, root: fx.targetRoot }], + plugins: [{ name: "stub", agents: [stub] }], + }), + ); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + await processProject({ + projectId: fx.projectId, + agentType: "stub", + concurrency: 1, + filePaths: ["held.ts", "free.ts"], + }); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0][0]).toContain("held.ts"); + expect(warn.mock.calls[0][0]).not.toContain("free.ts"); + } finally { + warn.mockRestore(); + } + const investigated = stub.calls.investigateCalls.flatMap((c) => c.batch.map((r) => r.filePath)); + expect(investigated).toEqual(["free.ts"]); + }); + it("process() captures refusals from the agent into AnalysisEntry", async () => { const fx = setupProject({ files: ["app.ts"] }); fx.writeRecord(pendingRecord(fx.projectId, "app.ts")); diff --git a/packages/processor/src/index.ts b/packages/processor/src/index.ts index 09693e95..0642195c 100644 --- a/packages/processor/src/index.ts +++ b/packages/processor/src/index.ts @@ -391,17 +391,6 @@ export async function process(params: { return ageMs >= STALE_LOCK_MS; }; - // True when a lock held by another run can be checked (it started on - // this host) and isn't reclaimable. Force mode leaves those files alone. - const isLiveLocalLock = (r: FileRecord): boolean => { - if (isReclaimableLock(r)) return false; - try { - return readRunMeta(projectId, r.lockedByRunId!).hostname === localHostname; - } catch { - return false; - } - }; - // Load file records and pick which to process const allRecords = loadAllFileRecords(projectId); let toProcess: FileRecord[]; @@ -537,6 +526,7 @@ export async function process(params: { // primitive and the 1h stale-lock cutoff. const lockedAt = new Date().toISOString(); const claimed: FileRecord[] = []; + const lockedPaths: string[] = []; const inForceMode = !!reinvestigate || params.filePaths !== undefined; const releaseProcessLock = await acquireProcessLock(projectId, runId); try { @@ -557,11 +547,11 @@ export async function process(params: { (current.status === "processing" && current.lockedByRunId !== runId && isReclaimableLock(current)); - // Force mode re-claims analyzed files and files locked by a run on - // another host, whose PID we can't probe. It leaves a file alone - // only while a run on this host holds a live lock on it, whatever - // the record's status. - if (!isOurs && !isFreelyClaimable && (!inForceMode || isLiveLocalLock(current))) { + // Force mode also re-claims analyzed files, so it can't rely on the + // status. It leaves a file alone only while another run's lock isn't + // reclaimable, whichever host that run is on. + if (!isOurs && !isFreelyClaimable && !(inForceMode && isReclaimableLock(current))) { + lockedPaths.push(current.filePath); continue; } @@ -578,6 +568,11 @@ export async function process(params: { } finally { releaseProcessLock(); } + if (params.filePaths !== undefined && lockedPaths.length > 0) { + console.warn( + `[deepsec] process: ${lockedPaths.length} file(s) were locked by another run and were skipped: ${lockedPaths.slice(0, 5).join(", ")}${lockedPaths.length > 5 ? "…" : ""}`, + ); + } toProcess = claimed; if (toProcess.length === 0) { emitProgress({ From 3f1055348acb90bb8a5a9ce3d85e76bb8641546a Mon Sep 17 00:00:00 2001 From: Terry <186405119+tlq5l@users.noreply.github.com> Date: Mon, 19 Oct 2026 23:48:30 +0700 Subject: [PATCH 4/8] Share the run metadata fixture in the lock tests The three lock tests each wrote the same run metadata file inline. A writeRun helper now writes it from the fields the tests vary: hostname, phase, and PID. --- .../src/__tests__/process-revalidate.test.ts | 69 +++++++------------ 1 file changed, 24 insertions(+), 45 deletions(-) diff --git a/packages/processor/src/__tests__/process-revalidate.test.ts b/packages/processor/src/__tests__/process-revalidate.test.ts index 67f5ada3..a98a19c5 100644 --- a/packages/processor/src/__tests__/process-revalidate.test.ts +++ b/packages/processor/src/__tests__/process-revalidate.test.ts @@ -76,6 +76,27 @@ function pendingRecord(projectId: string, filePath: string): FileRecord { }; } +function writeRun( + fx: Fixture, + runId: string, + meta: { hostname: string; phase: "running" | "done"; pid: number }, +): void { + const runsDir = path.join(fx.dataRoot, fx.projectId, "runs"); + fs.mkdirSync(runsDir, { recursive: true }); + fs.writeFileSync( + path.join(runsDir, `${runId}.json`), + JSON.stringify({ + runId, + projectId: fx.projectId, + rootPath: fx.targetRoot, + createdAt: new Date().toISOString(), + type: "process", + stats: {}, + ...meta, + }), + ); +} + describe("processor with stub agent", () => { let prevDataRoot: string | undefined; @@ -469,27 +490,13 @@ describe("processor with stub agent", () => { const deadRunId = "20260101000000-deadforceaaaaaaa"; const ownRunId = "20260101000000-ownforceaaaaaaaa"; - fs.mkdirSync(path.join(fx.dataRoot, fx.projectId, "runs"), { recursive: true }); const lock = (filePath: string, runId: string, pid: number) => { const rec = pendingRecord(fx.projectId, filePath); rec.status = "processing"; rec.lockedByRunId = runId; rec.lockedAt = new Date().toISOString(); fx.writeRecord(rec); - fs.writeFileSync( - path.join(fx.dataRoot, fx.projectId, "runs", `${runId}.json`), - JSON.stringify({ - runId, - projectId: fx.projectId, - rootPath: fx.targetRoot, - createdAt: new Date().toISOString(), - type: "process", - phase: "running", - pid, - hostname: os.hostname(), - stats: {}, - }), - ); + writeRun(fx, runId, { hostname: os.hostname(), phase: "running", pid }); }; lock("live.ts", liveRunId, process.pid); const liveBefore = fx.readRecord("live.ts"); @@ -552,26 +559,12 @@ describe("processor with stub agent", () => { const filePath = "locked.ts"; const fx = setupProject({ files: [filePath] }); const liveRunId = "20260101000000-liveremoteaaaaaa"; - fs.mkdirSync(path.join(fx.dataRoot, fx.projectId, "runs"), { recursive: true }); const rec = pendingRecord(fx.projectId, filePath); rec.status = status; rec.lockedByRunId = liveRunId; rec.lockedAt = new Date(Date.now() - ageMs).toISOString(); fx.writeRecord(rec); - fs.writeFileSync( - path.join(fx.dataRoot, fx.projectId, "runs", `${liveRunId}.json`), - JSON.stringify({ - runId: liveRunId, - projectId: fx.projectId, - rootPath: fx.targetRoot, - createdAt: new Date().toISOString(), - type: "process", - phase, - pid: process.pid, - hostname, - stats: {}, - }), - ); + writeRun(fx, liveRunId, { hostname, phase, pid: process.pid }); const before = fx.readRecord(filePath); const stub = new StubAgent(); @@ -596,27 +589,13 @@ describe("processor with stub agent", () => { it("process() with a file list warns about requested files another run holds", async () => { const fx = setupProject({ files: ["held.ts", "free.ts"] }); const otherRunId = "20260101000000-otherwarnaaaaaaa"; - fs.mkdirSync(path.join(fx.dataRoot, fx.projectId, "runs"), { recursive: true }); const held = pendingRecord(fx.projectId, "held.ts"); held.status = "processing"; held.lockedByRunId = otherRunId; held.lockedAt = new Date().toISOString(); fx.writeRecord(held); fx.writeRecord(pendingRecord(fx.projectId, "free.ts")); - fs.writeFileSync( - path.join(fx.dataRoot, fx.projectId, "runs", `${otherRunId}.json`), - JSON.stringify({ - runId: otherRunId, - projectId: fx.projectId, - rootPath: fx.targetRoot, - createdAt: new Date().toISOString(), - type: "process", - phase: "running", - pid: process.pid, - hostname: "other-host", - stats: {}, - }), - ); + writeRun(fx, otherRunId, { hostname: "other-host", phase: "running", pid: process.pid }); const stub = new StubAgent(); setLoadedConfig( From 64f7f920f40f22bbff42ef46e1cff80ec5cde0ee Mon Sep 17 00:00:00 2001 From: Terry <186405119+tlq5l@users.noreply.github.com> Date: Mon, 19 Oct 2026 23:48:30 +0700 Subject: [PATCH 5/8] Test that a force-mode claim keeps another run's newer findings The test holds the claim mutex while process() loads its snapshot, then lets another run finish the file and clear its lock. The claim must queue the record it re-read, not the snapshot, or the run writes the snapshot back and drops the other run's findings and history. --- .../src/__tests__/process-revalidate.test.ts | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/packages/processor/src/__tests__/process-revalidate.test.ts b/packages/processor/src/__tests__/process-revalidate.test.ts index a98a19c5..232c4d9e 100644 --- a/packages/processor/src/__tests__/process-revalidate.test.ts +++ b/packages/processor/src/__tests__/process-revalidate.test.ts @@ -622,6 +622,74 @@ describe("processor with stub agent", () => { expect(investigated).toEqual(["free.ts"]); }); + it("process() with --reinvestigate keeps findings another run wrote after the snapshot loaded", async () => { + // The other run's lock is live when the records are loaded, so the + // snapshot has no findings. The run finishes the file before the claim, + // which clears the lock and makes the file claimable. + const filePath = "shared.ts"; + const fx = setupProject({ files: [filePath] }); + const otherRunId = "20260101000000-othersnapaaaaaaa"; + const rec = pendingRecord(fx.projectId, filePath); + rec.status = "processing"; + rec.lockedByRunId = otherRunId; + rec.lockedAt = new Date().toISOString(); + fx.writeRecord(rec); + writeRun(fx, otherRunId, { hostname: os.hostname(), phase: "running", pid: process.pid }); + + const stub = new StubAgent(); + setLoadedConfig( + defineConfig({ + projects: [{ id: fx.projectId, root: fx.targetRoot }], + plugins: [{ name: "stub", agents: [stub] }], + }), + ); + + // Hold the claim mutex so process() loads its snapshot, then waits. + const lockDir = path.join(fx.dataRoot, fx.projectId, ".process.lock"); + fs.mkdirSync(lockDir); + const run = processProject({ + projectId: fx.projectId, + agentType: "stub", + concurrency: 1, + reinvestigate: true, + }); + + const finished = fx.readRecord(filePath); + finished.status = "analyzed"; + finished.lockedByRunId = undefined; + finished.lockedAt = undefined; + finished.findings = [ + { + severity: "HIGH", + vulnSlug: "auth-bypass", + title: "finding from the other run", + description: "written after the snapshot loaded", + lineNumbers: [1], + recommendation: "fix it", + confidence: "high", + }, + ]; + finished.analysisHistory = [ + { + runId: otherRunId, + investigatedAt: new Date().toISOString(), + durationMs: 1, + agentType: "stub", + model: "stub", + modelConfig: {}, + findingCount: 1, + }, + ]; + fx.writeRecord(finished); + fs.rmSync(lockDir, { recursive: true }); + await run; + + const after = fx.readRecord(filePath); + expect(after.findings.map((f) => f.title)).toContain("finding from the other run"); + expect(after.analysisHistory.map((h) => h.runId)).toContain(otherRunId); + expect(after.analysisHistory).toHaveLength(2); + }); + it("process() captures refusals from the agent into AnalysisEntry", async () => { const fx = setupProject({ files: ["app.ts"] }); fx.writeRecord(pendingRecord(fx.projectId, "app.ts")); From 8c134df704b2e6c8e2b03e43cb27bc09c335ee40 Mon Sep 17 00:00:00 2001 From: "Terry (agent)" <186405119+tlq5l@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:05:10 +0700 Subject: [PATCH 6/8] Count --limit by claimed files and warn on skipped locks in every mode --- packages/processor/src/index.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/processor/src/index.ts b/packages/processor/src/index.ts index 0642195c..14d91f23 100644 --- a/packages/processor/src/index.ts +++ b/packages/processor/src/index.ts @@ -510,11 +510,6 @@ export async function process(params: { toProcess = toProcess.filter((r) => r.filePath.startsWith(params.filter!)); } - // Apply limit - if (params.limit && toProcess.length > params.limit) { - toProcess = toProcess.slice(0, params.limit); - } - // Atomic claim under a per-project mutex. Without serializing this // section, two `process()` invocations against the same project both // see the same pending records, both write status="processing" with @@ -531,6 +526,8 @@ export async function process(params: { const releaseProcessLock = await acquireProcessLock(projectId, runId); try { for (const record of toProcess) { + // The limit counts claimed files, so locked ones don't use up slots. + if (params.limit && claimed.length >= params.limit) break; let current: FileRecord; try { const fresh = readFileRecord(projectId, record.filePath); @@ -568,7 +565,7 @@ export async function process(params: { } finally { releaseProcessLock(); } - if (params.filePaths !== undefined && lockedPaths.length > 0) { + if (lockedPaths.length > 0) { console.warn( `[deepsec] process: ${lockedPaths.length} file(s) were locked by another run and were skipped: ${lockedPaths.slice(0, 5).join(", ")}${lockedPaths.length > 5 ? "…" : ""}`, ); From f3105485d69170c5050278aa6e068227bbb7276b Mon Sep 17 00:00:00 2001 From: Terry <186405119+tlq5l@users.noreply.github.com> Date: Wed, 21 Oct 2026 13:46:53 +0700 Subject: [PATCH 7/8] Check the lock before status, expire bad lock timestamps, fail direct mode on skipped files A fresh lock held by another run now blocks a claim whatever the record status (pending and error included). An unparseable or future lockedAt is treated as reclaimable. process() returns skippedCount, and direct mode exits 1 when it is non-zero so a forged or live lock can't pass the CI gate. Agent: claude --- .../src/__tests__/setup-coordinator.test.ts | 1 + packages/deepsec/src/commands/process.ts | 9 +++ packages/deepsec/src/setup/coordinator.ts | 1 + .../src/__tests__/process-revalidate.test.ts | 58 ++++++++++++++++++- packages/processor/src/index.ts | 44 +++++++++----- 5 files changed, 98 insertions(+), 15 deletions(-) diff --git a/packages/deepsec/src/__tests__/setup-coordinator.test.ts b/packages/deepsec/src/__tests__/setup-coordinator.test.ts index f3b312c0..763845c3 100644 --- a/packages/deepsec/src/__tests__/setup-coordinator.test.ts +++ b/packages/deepsec/src/__tests__/setup-coordinator.test.ts @@ -155,6 +155,7 @@ describe("one-shot setup coordinator", () => { analysisCount: 1, findingCount: 0, errorBatchCount: 0, + skippedCount: 0, })); let fingerprint = "source-v1"; const services = { diff --git a/packages/deepsec/src/commands/process.ts b/packages/deepsec/src/commands/process.ts index 35c609c3..4182b747 100644 --- a/packages/deepsec/src/commands/process.ts +++ b/packages/deepsec/src/commands/process.ts @@ -388,6 +388,15 @@ async function processDirectMode(opts: Parameters[0]) { console.log(`${RED}${result.findingCount} new finding(s) — exiting 1${RESET}`); process.exit(1); } + // Files another run holds are unreviewed, not clean. A forged or stale + // lock must not turn the CI gate green, so fail rather than pass quietly. + if (result.skippedCount > 0) { + console.log(); + console.log( + `${RED}${result.skippedCount} file(s) skipped because another run holds their lock — exiting 1 (not a clean review).${RESET}`, + ); + process.exit(1); + } console.log(); console.log(`${GREEN}No findings.${RESET}`); } diff --git a/packages/deepsec/src/setup/coordinator.ts b/packages/deepsec/src/setup/coordinator.ts index 41f73ccb..378f2f60 100644 --- a/packages/deepsec/src/setup/coordinator.ts +++ b/packages/deepsec/src/setup/coordinator.ts @@ -868,6 +868,7 @@ export async function runSetupWorkflow( analysisCount: 0, findingCount: 0, errorBatchCount: 0, + skippedCount: 0, }; reporter.emit({ type: "phase-skip", diff --git a/packages/processor/src/__tests__/process-revalidate.test.ts b/packages/processor/src/__tests__/process-revalidate.test.ts index 232c4d9e..52c11a05 100644 --- a/packages/processor/src/__tests__/process-revalidate.test.ts +++ b/packages/processor/src/__tests__/process-revalidate.test.ts @@ -545,6 +545,30 @@ describe("processor with stub agent", () => { 0, true, ], + [ + "leaves a pending file with a fresh lock from another host", + "pending", + "other-host", + "running", + 0, + false, + ], + [ + "leaves an errored file with a fresh lock from another host", + "error", + "other-host", + "running", + 0, + false, + ], + [ + "takes a lock from another host dated a day in the future", + "processing", + "other-host", + "running", + -24 * 60 * 60_000, + true, + ], [ "leaves an analyzed file with a live lock on this host", "analyzed", @@ -574,7 +598,7 @@ describe("processor with stub agent", () => { plugins: [{ name: "stub", agents: [stub] }], }), ); - await processProject({ + const result = await processProject({ projectId: fx.projectId, agentType: "stub", concurrency: 1, @@ -583,9 +607,38 @@ describe("processor with stub agent", () => { const investigated = stub.calls.investigateCalls.flatMap((c) => c.batch.map((r) => r.filePath)); expect(investigated).toEqual(taken ? [filePath] : []); + expect(result.skippedCount).toBe(taken ? 0 : 1); if (!taken) expect(fx.readRecord(filePath)).toEqual(before); }); + it("process() with --reinvestigate takes a lock whose timestamp is unparseable", async () => { + const fx = setupProject({ files: ["locked.ts"] }); + const liveRunId = "20260101000000-badstampaaaaaaaa"; + const rec = pendingRecord(fx.projectId, "locked.ts"); + rec.status = "processing"; + rec.lockedByRunId = liveRunId; + rec.lockedAt = "not-a-date"; + fx.writeRecord(rec); + writeRun(fx, liveRunId, { hostname: "other-host", phase: "running", pid: process.pid }); + + const stub = new StubAgent(); + setLoadedConfig( + defineConfig({ + projects: [{ id: fx.projectId, root: fx.targetRoot }], + plugins: [{ name: "stub", agents: [stub] }], + }), + ); + await processProject({ + projectId: fx.projectId, + agentType: "stub", + concurrency: 1, + reinvestigate: true, + }); + + const investigated = stub.calls.investigateCalls.flatMap((c) => c.batch.map((r) => r.filePath)); + expect(investigated).toEqual(["locked.ts"]); + }); + it("process() with a file list warns about requested files another run holds", async () => { const fx = setupProject({ files: ["held.ts", "free.ts"] }); const otherRunId = "20260101000000-otherwarnaaaaaaa"; @@ -606,12 +659,13 @@ describe("processor with stub agent", () => { ); const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); try { - await processProject({ + const result = await processProject({ projectId: fx.projectId, agentType: "stub", concurrency: 1, filePaths: ["held.ts", "free.ts"], }); + expect(result.skippedCount).toBe(1); expect(warn).toHaveBeenCalledTimes(1); expect(warn.mock.calls[0][0]).toContain("held.ts"); expect(warn.mock.calls[0][0]).not.toContain("free.ts"); diff --git a/packages/processor/src/index.ts b/packages/processor/src/index.ts index 14d91f23..f13ff371 100644 --- a/packages/processor/src/index.ts +++ b/packages/processor/src/index.ts @@ -164,6 +164,12 @@ export async function process(params: { * crashed CLI). CI gates on this so a silent fail doesn't pass. */ errorBatchCount: number; + /** + * Files left unclaimed because another run held a live lock on them. A + * run that requested files but claimed none is not a clean review, so + * direct mode (the CI gate) fails on a non-zero count. + */ + skippedCount: number; /** * Set when one of the batches threw `QuotaExhaustedError`. Once any * batch hits this, every subsequent API call against the same credential @@ -293,7 +299,7 @@ export async function process(params: { type: "all_complete", message: `Run ${runId} already completed`, }); - return { runId, analysisCount: 0, findingCount: 0, errorBatchCount: 0 }; + return { runId, analysisCount: 0, findingCount: 0, errorBatchCount: 0, skippedCount: 0 }; } } else { // Create new run @@ -354,6 +360,7 @@ export async function process(params: { // thinking. False reclaims are catastrophic; false rejections only // cost a retry on the next run. const STALE_LOCK_MS = 60 * 60 * 1000; + const FUTURE_SKEW_MS = 5 * 60 * 1000; const localHostname = os.hostname(); const isReclaimableLock = (r: FileRecord): boolean => { if (!r.lockedByRunId) return true; @@ -387,8 +394,11 @@ export async function process(params: { // those as old enough to reclaim so we can recover legacy locked // state. if (!r.lockedAt) return true; - const ageMs = Date.now() - new Date(r.lockedAt).getTime(); - return ageMs >= STALE_LOCK_MS; + // An unparseable or future timestamp would never age out, so it + // can't hold a lock. FUTURE_SKEW_MS allows for clock drift between hosts. + const lockedMs = Date.parse(r.lockedAt); + if (Number.isNaN(lockedMs) || lockedMs > Date.now() + FUTURE_SKEW_MS) return true; + return Date.now() - lockedMs >= STALE_LOCK_MS; }; // Load file records and pick which to process @@ -502,7 +512,7 @@ export async function process(params: { message: "No files to process", }); completeRun(projectId, runId, "done", { filesProcessed: 0 }); - return { runId, analysisCount: 0, findingCount: 0, errorBatchCount: 0 }; + return { runId, analysisCount: 0, findingCount: 0, errorBatchCount: 0, skippedCount: 0 }; } // Apply path filter @@ -538,16 +548,17 @@ export async function process(params: { } const isOurs = current.lockedByRunId === runId; - const isFreelyClaimable = + // Another run's lock decides first, whatever the status says: a fresh + // lock on a `pending` or `error` record is still that run's file. + // Otherwise a pending/error/processing record is free to take, and + // force mode also takes analyzed ones. + const heldByOther = !isOurs && !isReclaimableLock(current); + const isClaimable = + inForceMode || current.status === "pending" || current.status === "error" || - (current.status === "processing" && - current.lockedByRunId !== runId && - isReclaimableLock(current)); - // Force mode also re-claims analyzed files, so it can't rely on the - // status. It leaves a file alone only while another run's lock isn't - // reclaimable, whichever host that run is on. - if (!isOurs && !isFreelyClaimable && !(inForceMode && isReclaimableLock(current))) { + current.status === "processing"; + if (!isOurs && (heldByOther || !isClaimable)) { lockedPaths.push(current.filePath); continue; } @@ -577,7 +588,13 @@ export async function process(params: { message: "Nothing to claim — another run owned every candidate file.", }); completeRun(projectId, runId, "done", { filesProcessed: 0 }); - return { runId, analysisCount: 0, findingCount: 0, errorBatchCount: 0 }; + return { + runId, + analysisCount: 0, + findingCount: 0, + errorBatchCount: 0, + skippedCount: lockedPaths.length, + }; } const batches = batchCandidates(toProcess, params.batchSize); @@ -856,6 +873,7 @@ export async function process(params: { analysisCount: totalAnalyses, findingCount: totalFindings, errorBatchCount: batchesFailed, + skippedCount: lockedPaths.length, quotaExhausted, totalCostUsd, costLimitReached, From cbb0f7d9f1ce4a60e587ed1aa4ef8f97527482c0 Mon Sep 17 00:00:00 2001 From: Terry <186405119+tlq5l@users.noreply.github.com> Date: Wed, 21 Oct 2026 13:52:23 +0700 Subject: [PATCH 8/8] Report another run's processing files as skipped in default mode The default-mode selector dropped processing records with a live foreign lock before the claim step, so they were never warned about or counted in skippedCount. Pass them to the claim step, which already skips and counts locks it can't reclaim. Agent: claude --- .../src/__tests__/process-revalidate.test.ts | 37 +++++++++++++++++++ packages/processor/src/index.ts | 10 ++--- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/packages/processor/src/__tests__/process-revalidate.test.ts b/packages/processor/src/__tests__/process-revalidate.test.ts index 52c11a05..a0dfbb61 100644 --- a/packages/processor/src/__tests__/process-revalidate.test.ts +++ b/packages/processor/src/__tests__/process-revalidate.test.ts @@ -676,6 +676,43 @@ describe("processor with stub agent", () => { expect(investigated).toEqual(["free.ts"]); }); + it("process() in default mode warns about a processing file another run holds", async () => { + const fx = setupProject({ files: ["held.ts", "free.ts"] }); + const otherRunId = "20260101000000-otherdefaultaaaa"; + const held = pendingRecord(fx.projectId, "held.ts"); + held.status = "processing"; + held.lockedByRunId = otherRunId; + held.lockedAt = new Date().toISOString(); + fx.writeRecord(held); + fx.writeRecord(pendingRecord(fx.projectId, "free.ts")); + writeRun(fx, otherRunId, { hostname: "other-host", phase: "running", pid: process.pid }); + const before = fx.readRecord("held.ts"); + + const stub = new StubAgent(); + setLoadedConfig( + defineConfig({ + projects: [{ id: fx.projectId, root: fx.targetRoot }], + plugins: [{ name: "stub", agents: [stub] }], + }), + ); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + const result = await processProject({ + projectId: fx.projectId, + agentType: "stub", + concurrency: 1, + }); + expect(result.skippedCount).toBe(1); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0][0]).toContain("held.ts"); + } finally { + warn.mockRestore(); + } + const investigated = stub.calls.investigateCalls.flatMap((c) => c.batch.map((r) => r.filePath)); + expect(investigated).toEqual(["free.ts"]); + expect(fx.readRecord("held.ts")).toEqual(before); + }); + it("process() with --reinvestigate keeps findings another run wrote after the snapshot loaded", async () => { // The other run's lock is live when the records are loaded, so the // snapshot has no findings. The run finishes the file before the claim, diff --git a/packages/processor/src/index.ts b/packages/processor/src/index.ts index f13ff371..abe5a5c7 100644 --- a/packages/processor/src/index.ts +++ b/packages/processor/src/index.ts @@ -455,12 +455,10 @@ export async function process(params: { (r) => r.status === "pending" || r.status === "error" || - // Reclaim a `processing` record from another run only when the - // owning lock is genuinely abandoned. The previous version - // reclaimed unconditionally, which let two concurrent runs - // both pick up the same file and clobber each other on - // write. See `isReclaimableLock` for the exact criteria. - (r.status === "processing" && r.lockedByRunId !== runId && isReclaimableLock(r)), + // Another run's `processing` record goes on to the claim step, + // which takes it only if the lock is reclaimable and otherwise + // reports it as skipped. Dropping it here would hide it. + (r.status === "processing" && r.lockedByRunId !== runId), ); }