diff --git a/CHANGELOG.md b/CHANGELOG.md index bde3018c..57333592 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.27.0 + +Saved articles retain the evidence behind their metadata. PDF capture improves reading order and reports uncertain layout. + +- Keep observed titles, authors, dates, descriptions, standard page tags and bounded JSON-LD claims with their sources and selection rules. Dates without a timezone remain explicitly uncertain. +- Read separated PDF columns in order, preserve simple aligned tables and basic superscripts, and report layout losses as partial captures. Keep the original PDF for reference. +- Add public PDF bundle inspection and verification for source, Markdown, page references and image hashes, with bounded reads and typed capture errors. + ## 0.26.3 SDK consumers can augment Node environment types without breaking isolated subprocesses. diff --git a/README.md b/README.md index ac48ed69..47a150fa 100644 --- a/README.md +++ b/README.md @@ -74,14 +74,14 @@ Wordcell keeps the record in Markdown files you own, rebuilds every index from t The CLI and TypeScript SDK run with Bun. Install the versioned GitHub archive: ```sh -bun add --global --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.26.3/hraness-wordcell-0.26.3.tgz +bun add --global --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.27.0/hraness-wordcell-0.27.0.tgz wordcell --help ``` Prefer npm? The same release is [mirrored there](https://www.npmjs.com/package/@hraness/wordcell): ```sh -npm install --global --ignore-scripts @hraness/wordcell@0.26.3 +npm install --global --ignore-scripts @hraness/wordcell@0.27.0 wordcell --help ``` @@ -145,7 +145,7 @@ After trying the CLI, install the public Agent Skill into a compatible agent, such as Claude Code, Codex, Cursor, or GitHub Copilot: ```sh -bunx skills add hraness/wordcell#v0.26.3 --skill wordcell +bunx skills add hraness/wordcell#v0.27.0 --skill wordcell ``` Then ask: @@ -361,7 +361,7 @@ exist. [Graph queries and proof limits](https://github.com/hraness/wordcell/blob Add the same immutable release to a Bun project: ```sh -bun add --exact --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.26.3/hraness-wordcell-0.26.3.tgz +bun add --exact --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.27.0/hraness-wordcell-0.27.0.tgz ``` The SDK provides read-only vault sessions, metadata queries, search, graph diff --git a/bun.lock b/bun.lock index f92b3ba6..3e180bc3 100644 --- a/bun.lock +++ b/bun.lock @@ -13,6 +13,7 @@ "agent-browser": "0.32.3", "defuddle": "0.19.1", "effect": "3.22.1", + "entities": "7.0.1", "yaml": "^2.9.0", "zod": "^4.4.3", }, diff --git a/dist/capture.js b/dist/capture.js index 62036e88..00014340 100644 --- a/dist/capture.js +++ b/dist/capture.js @@ -5,19 +5,23 @@ import { captureSummary, main, runCapture -} from "./index-h9vyd03z.js"; -import"./index-hacpnaew.js"; -import"./index-k86wepd8.js"; +} from "./index-58azscg5.js"; +import"./index-fd8j527z.js"; +import"./index-h2w7syy2.js"; import { adapterCapabilities, inspectClipEnvironment, renderDoctorReport -} from "./index-nd8v7r0z.js"; +} from "./index-sp43012q.js"; import"./index-hnvx7m1j.js"; -import"./index-ncwzmsge.js"; -import"./index-cd75vky9.js"; +import"./index-nwdzhj1h.js"; +import"./index-jbtyp6q5.js"; import"./index-2gv8y733.js"; -import"./index-mxxxytys.js"; +import"./index-g63d3937.js"; +import { + extractArticleMetadata, + parseArticleMetadata +} from "./index-e2wstc7t.js"; import"./index-w2zc0vwa.js"; import"./index-e5fbsywq.js"; import"./index-gh719d91.js"; @@ -38,11 +42,15 @@ var inspectClipEnvironment2 = (...arguments_) => inspectClipEnvironment(...argum var parseCaptureArguments = (...arguments_) => parseArguments(...arguments_); var renderDoctorReport2 = (...arguments_) => renderDoctorReport(...arguments_); var runCapture2 = (...arguments_) => runCapture(...arguments_); +var extractArticleMetadata2 = (...arguments_) => extractArticleMetadata(...arguments_); +var parseArticleMetadata2 = (...arguments_) => parseArticleMetadata(...arguments_); export { runCapture2 as runCapture, renderDoctorReport2 as renderDoctorReport, parseCaptureArguments, + parseArticleMetadata2 as parseArticleMetadata, inspectClipEnvironment2 as inspectClipEnvironment, + extractArticleMetadata2 as extractArticleMetadata, clipMain, captureSummary2 as captureSummary, captureSucceeded2 as captureSucceeded, diff --git a/dist/cli-bin.js b/dist/cli-bin.js index 04a7ff97..daf19df8 100755 --- a/dist/cli-bin.js +++ b/dist/cli-bin.js @@ -2,7 +2,7 @@ // @bun import { runWordcellBin -} from "./index-6c8xhred.js"; +} from "./index-k9yzc4vn.js"; import"./index-3t0v457d.js"; import"./index-t4hjm1kk.js"; import"./index-4knsp9qj.js"; diff --git a/dist/cli.js b/dist/cli.js index 50317080..78a5d35a 100755 --- a/dist/cli.js +++ b/dist/cli.js @@ -6,22 +6,23 @@ import { parseArguments, runExecutable, usage -} from "./index-878bh2f1.js"; -import"./index-5h5awwh4.js"; -import"./index-m9p2a4mj.js"; +} from "./index-30v9w4cy.js"; +import"./index-b40e2dkx.js"; +import"./index-my50xsa3.js"; import"./index-3hts6yy9.js"; -import"./index-j4zgmzjr.js"; -import"./index-npg9z1a4.js"; -import"./index-h9vyd03z.js"; -import"./index-hacpnaew.js"; -import"./index-k86wepd8.js"; -import"./index-nd8v7r0z.js"; +import"./index-8djdwjjd.js"; +import"./index-xn5q4kne.js"; +import"./index-58azscg5.js"; +import"./index-fd8j527z.js"; +import"./index-h2w7syy2.js"; +import"./index-sp43012q.js"; import"./index-hnvx7m1j.js"; import"./index-t4hjm1kk.js"; -import"./index-ncwzmsge.js"; -import"./index-cd75vky9.js"; +import"./index-nwdzhj1h.js"; +import"./index-jbtyp6q5.js"; import"./index-2gv8y733.js"; -import"./index-mxxxytys.js"; +import"./index-g63d3937.js"; +import"./index-e2wstc7t.js"; import"./index-w2zc0vwa.js"; import"./index-e5fbsywq.js"; import"./index-gh719d91.js"; @@ -65,7 +66,7 @@ import { // src/cli.ts async function runStandaloneCli(args = process.argv.slice(2)) { - const { standaloneSupportEnvironment, isUsefulSupportResult, runProductSupportCommand, showProductSupportInvitation } = await import("./support-hd4754m4.js"); + const { standaloneSupportEnvironment, isUsefulSupportResult, runProductSupportCommand, showProductSupportInvitation } = await import("./support-s059eytv.js"); const env = standaloneSupportEnvironment(); if (args[0] === "support") { return await runProductSupportCommand(args.slice(1), { env }); diff --git a/dist/clip/acquire.js b/dist/clip/acquire.js index 619e417c..240f4c0a 100644 --- a/dist/clip/acquire.js +++ b/dist/clip/acquire.js @@ -22,10 +22,11 @@ import { mergeRenderedTextSnapshots, readBrowserExpansionTelemetry, seedOwnedBrowserCookies -} from "../index-ncwzmsge.js"; -import"../index-cd75vky9.js"; +} from "../index-nwdzhj1h.js"; +import"../index-jbtyp6q5.js"; import"../index-2gv8y733.js"; -import"../index-mxxxytys.js"; +import"../index-g63d3937.js"; +import"../index-e2wstc7t.js"; import"../index-w2zc0vwa.js"; import"../index-e5fbsywq.js"; import"../index-gh719d91.js"; diff --git a/dist/clip/bundle-reader.js b/dist/clip/bundle-reader.js index 4b1a1036..63853325 100644 --- a/dist/clip/bundle-reader.js +++ b/dist/clip/bundle-reader.js @@ -2,8 +2,9 @@ import { readCaptureBundle, verifyCaptureBundle -} from "../index-npg9z1a4.js"; -import"../index-mxxxytys.js"; +} from "../index-xn5q4kne.js"; +import"../index-g63d3937.js"; +import"../index-e2wstc7t.js"; import"../index-1xxnjn0d.js"; import"../index-z1w83f81.js"; export { diff --git a/dist/clip/cli.js b/dist/clip/cli.js index 4363f2ec..05ff0729 100755 --- a/dist/clip/cli.js +++ b/dist/clip/cli.js @@ -5,15 +5,16 @@ import { captureSucceeded, captureSummary, main -} from "../index-h9vyd03z.js"; -import"../index-hacpnaew.js"; -import"../index-k86wepd8.js"; -import"../index-nd8v7r0z.js"; +} from "../index-58azscg5.js"; +import"../index-fd8j527z.js"; +import"../index-h2w7syy2.js"; +import"../index-sp43012q.js"; import"../index-hnvx7m1j.js"; -import"../index-ncwzmsge.js"; -import"../index-cd75vky9.js"; +import"../index-nwdzhj1h.js"; +import"../index-jbtyp6q5.js"; import"../index-2gv8y733.js"; -import"../index-mxxxytys.js"; +import"../index-g63d3937.js"; +import"../index-e2wstc7t.js"; import"../index-w2zc0vwa.js"; import"../index-e5fbsywq.js"; import"../index-gh719d91.js"; diff --git a/dist/clip/defuddle-worker.js b/dist/clip/defuddle-worker.js index aaab7313..d6882af7 100644 --- a/dist/clip/defuddle-worker.js +++ b/dist/clip/defuddle-worker.js @@ -1,4 +1,7 @@ // @bun +import { + extractArticleMetadata +} from "../index-e2wstc7t.js"; import"../index-z1w83f81.js"; // src/clip/defuddle-worker.ts @@ -161,7 +164,7 @@ workerGlobal.onmessage = async (event) => { useAsync: false, removeImages: false }); - const enriched = typeof value === "object" && value !== null && !Array.isArray(value) ? { ...value, captureVideoPosters: videoPosters } : value; + const enriched = typeof value === "object" && value !== null && !Array.isArray(value) ? { ...value, captureVideoPosters: videoPosters, metadata: await extractArticleMetadata(request.html, request.url, value) } : value; workerGlobal.postMessage({ ok: true, value: enriched }); } catch { workerGlobal.postMessage({ ok: false, message: "Defuddle could not parse this acquisition." }); diff --git a/dist/clip/doctor.js b/dist/clip/doctor.js index 8b25b8b8..232d54b7 100644 --- a/dist/clip/doctor.js +++ b/dist/clip/doctor.js @@ -9,11 +9,12 @@ import { renderAdapterCapabilities, renderDoctorReport, runDiagnosticCommand -} from "../index-nd8v7r0z.js"; -import"../index-ncwzmsge.js"; -import"../index-cd75vky9.js"; +} from "../index-sp43012q.js"; +import"../index-nwdzhj1h.js"; +import"../index-jbtyp6q5.js"; import"../index-2gv8y733.js"; -import"../index-mxxxytys.js"; +import"../index-g63d3937.js"; +import"../index-e2wstc7t.js"; import"../index-w2zc0vwa.js"; import"../index-e5fbsywq.js"; import"../index-gh719d91.js"; diff --git a/dist/clip/jobs.js b/dist/clip/jobs.js index d9aac11b..8747c658 100644 --- a/dist/clip/jobs.js +++ b/dist/clip/jobs.js @@ -1,7 +1,8 @@ // @bun import { redactSensitiveText -} from "../index-mxxxytys.js"; +} from "../index-g63d3937.js"; +import"../index-e2wstc7t.js"; import { sanitizeTerminalLine, sanitizeTerminalText diff --git a/dist/clip/persist.js b/dist/clip/persist.js index f137d25c..3938b48b 100644 --- a/dist/clip/persist.js +++ b/dist/clip/persist.js @@ -12,7 +12,8 @@ import { sanitizeArtifactUrl, sanitizeSourceHtml, writeCaptureBundle -} from "../index-mxxxytys.js"; +} from "../index-g63d3937.js"; +import"../index-e2wstc7t.js"; import"../index-1xxnjn0d.js"; import"../index-z1w83f81.js"; export { diff --git a/dist/clip/refresh.js b/dist/clip/refresh.js index 9b5eebef..cb8fb8cc 100644 --- a/dist/clip/refresh.js +++ b/dist/clip/refresh.js @@ -1,9 +1,10 @@ // @bun import { diffCaptureBundle -} from "../index-j4zgmzjr.js"; -import"../index-npg9z1a4.js"; -import"../index-mxxxytys.js"; +} from "../index-8djdwjjd.js"; +import"../index-xn5q4kne.js"; +import"../index-g63d3937.js"; +import"../index-e2wstc7t.js"; import"../index-1gwbassd.js"; import"../index-1xxnjn0d.js"; import"../index-z1w83f81.js"; diff --git a/dist/evaluation-builder-bin.js b/dist/evaluation-builder-bin.js index 5c734861..57475e66 100755 --- a/dist/evaluation-builder-bin.js +++ b/dist/evaluation-builder-bin.js @@ -2,7 +2,7 @@ // @bun import { runWordcellBin -} from "./index-6c8xhred.js"; +} from "./index-k9yzc4vn.js"; import"./index-3t0v457d.js"; import"./index-t4hjm1kk.js"; import"./index-4knsp9qj.js"; diff --git a/dist/index-878bh2f1.js b/dist/index-30v9w4cy.js similarity index 99% rename from dist/index-878bh2f1.js rename to dist/index-30v9w4cy.js index f2ba02cb..f04f5c82 100644 --- a/dist/index-878bh2f1.js +++ b/dist/index-30v9w4cy.js @@ -3,10 +3,10 @@ import { backfillSavedUrlMetadata, createRustMetadataSearchProvider, savedSourceProblem -} from "./index-5h5awwh4.js"; +} from "./index-b40e2dkx.js"; import { main as main2 -} from "./index-m9p2a4mj.js"; +} from "./index-my50xsa3.js"; import { MAX_AUTHORIZED_VAULTS, auditKnowledgePortfolio, @@ -16,13 +16,13 @@ import { } from "./index-3hts6yy9.js"; import { diffCaptureBundle -} from "./index-j4zgmzjr.js"; +} from "./index-8djdwjjd.js"; import { verifyCaptureBundle -} from "./index-npg9z1a4.js"; +} from "./index-xn5q4kne.js"; import { main -} from "./index-h9vyd03z.js"; +} from "./index-58azscg5.js"; import { advancedHelp, closestMatch, @@ -43,11 +43,11 @@ import { symbol, terminalOutput, terminalStyle -} from "./index-cd75vky9.js"; +} from "./index-jbtyp6q5.js"; import { redactSensitiveText, sanitizeArtifactUrl -} from "./index-mxxxytys.js"; +} from "./index-g63d3937.js"; import { initVault } from "./index-23z4zxgg.js"; diff --git a/dist/index-h9vyd03z.js b/dist/index-58azscg5.js similarity index 99% rename from dist/index-h9vyd03z.js rename to dist/index-58azscg5.js index 3adf068f..240bd3e3 100644 --- a/dist/index-h9vyd03z.js +++ b/dist/index-58azscg5.js @@ -2,7 +2,7 @@ import { acquireArchiveTodaySnapshot, discoverArchiveTodaySnapshot -} from "./index-hacpnaew.js"; +} from "./index-fd8j527z.js"; import { canonicalizeUrl, chooseBestExtraction, @@ -12,13 +12,13 @@ import { localizeAssets, scoreExtraction, sniffImage -} from "./index-k86wepd8.js"; +} from "./index-h2w7syy2.js"; import { adapterCapabilities, inspectClipEnvironment, renderAdapterCapabilities, renderDoctorReport -} from "./index-nd8v7r0z.js"; +} from "./index-sp43012q.js"; import { cloneBrowserProfile } from "./index-hnvx7m1j.js"; @@ -34,7 +34,7 @@ import { renderCookiePermissionNotice, renderCookiePermissionRecovery, setCookiePermissionReporter -} from "./index-ncwzmsge.js"; +} from "./index-nwdzhj1h.js"; import { CONTENT_REWRITE_TRUNCATION_WARNING, buildClipMarkdown, @@ -51,7 +51,7 @@ import { stderrStyle, terminalOutput, writesToTerminal -} from "./index-cd75vky9.js"; +} from "./index-jbtyp6q5.js"; import { MAX_COOKIE_BYTES, filterCookieProviderResult, @@ -68,7 +68,7 @@ import { redactSensitiveTextWithCount, sanitizeArtifactUrl, writeCaptureBundle -} from "./index-mxxxytys.js"; +} from "./index-g63d3937.js"; import { startNetworkProxy } from "./index-w2zc0vwa.js"; @@ -2285,6 +2285,7 @@ async function runCapture(rawOptions, dependencies = {}) { const finalWarnings = finalizedWarnings(combinedWarnings, redactedMarkdown.count); const includeSource = options.evidence === "source" || options.evidence === "all"; const manifestInput = { + ...article.metadata === undefined ? {} : { metadata: article.metadata }, sourceUrl, canonicalUrl: best.canonicalUrl.href, capturedAt, diff --git a/dist/index-j4zgmzjr.js b/dist/index-8djdwjjd.js similarity index 99% rename from dist/index-j4zgmzjr.js rename to dist/index-8djdwjjd.js index 1fb5c360..a262dfc3 100644 --- a/dist/index-j4zgmzjr.js +++ b/dist/index-8djdwjjd.js @@ -1,7 +1,7 @@ // @bun import { readCaptureBundle -} from "./index-npg9z1a4.js"; +} from "./index-xn5q4kne.js"; import { runGitCommand } from "./index-1gwbassd.js"; diff --git a/dist/index-5h5awwh4.js b/dist/index-b40e2dkx.js similarity index 99% rename from dist/index-5h5awwh4.js rename to dist/index-b40e2dkx.js index 4a28c50a..fccf60d4 100644 --- a/dist/index-5h5awwh4.js +++ b/dist/index-b40e2dkx.js @@ -5,10 +5,10 @@ import { parseArchiveTodayMementoUrl, parseMetadataSearchResponse, rankMetadataSearchResults -} from "./index-hacpnaew.js"; +} from "./index-fd8j527z.js"; import { sanitizeArtifactUrl -} from "./index-mxxxytys.js"; +} from "./index-g63d3937.js"; import { assertSafeNetworkUrl, isPrivateHostname, diff --git a/dist/index-e2wstc7t.js b/dist/index-e2wstc7t.js new file mode 100644 index 00000000..f9c8b51b --- /dev/null +++ b/dist/index-e2wstc7t.js @@ -0,0 +1,323 @@ +// @bun +// src/clip/metadata.ts +import { decodeHTML } from "entities"; +var metadataLimits = { htmlBytes: 8 * 1024 * 1024, claims: 256, value: 8192, totalBytes: 256 * 1024, jsonBytes: 128 * 1024, jsonNodes: 2048, jsonDepth: 12 }; +var record = (v) => typeof v === "object" && v !== null && !Array.isArray(v); +var sources = new Set(["html-title", "html-meta", "html-link", "html-language", "json-ld", "extractor"]); +var warningKinds = new Set(["input-limit", "claim-limit", "value-limit", "json-limit", "json-invalid"]); +var credentialName = /(?:csrf|xsrf|token|nonce|password|secret|session|authorization|api[-_]?key|signature)/i; +var urlFields = new Set(["canonical", "alternate", "image", "url"]); +var dateFields = new Set(["published", "modified"]); +function publicUrl(value, base) { + try { + const url = new URL(value, base); + if (!/^https?:$/.test(url.protocol)) + return null; + url.username = ""; + url.password = ""; + url.search = ""; + url.hash = ""; + return url.href; + } catch { + return null; + } +} +function cleanText(value) { + return value.replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f-\u009f]/gu, "").replace(/https?:\/\/[^\s<>"']+/giu, (url) => publicUrl(url) ?? "[invalid URL]").replace(/\s+/gu, " ").trim(); +} +function dateValue(value) { + const dateOnly = /^\d{4}-\d{2}-\d{2}$/.test(value); + const zoned = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d{1,9})?)?(?:Z|[+-]\d{2}:\d{2})$/i.test(value); + const local = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d{1,9})?)?$/.test(value); + const parsed = Date.parse(dateOnly ? value + "T00:00:00Z" : local ? value + "Z" : value); + const validDay = Number.isFinite(parsed) && new Date(Date.UTC(Number(value.slice(0, 4)), Number(value.slice(5, 7)) - 1, Number(value.slice(8, 10)))).toISOString().slice(0, 10) === value.slice(0, 10); + if (!dateOnly && !zoned && !local || !validDay) + return { value, datePrecision: "invalid", timezone: "unknown" }; + if (dateOnly) + return { value, datePrecision: "date", timezone: "not-applicable" }; + return { value: zoned ? new Date(parsed).toISOString() : value, datePrecision: "datetime", timezone: zoned ? "explicit" : "unknown" }; +} +function rank(claim) { + if (claim.datePrecision === "invalid") + return 1000; + if (claim.source === "json-ld") + return claim.rule.includes("jsonld-type=WebPage") ? 15 : 10; + if (claim.source === "html-link") + return 5; + if (claim.locator.includes("property=article:")) + return 20; + if (claim.locator.includes("property=og:")) + return 30; + if (claim.locator.includes("name=twitter:")) + return 40; + if (claim.source === "html-meta") + return 50; + if (claim.source === "html-title" || claim.source === "html-language") + return 60; + return 90; +} +function select(claims) { + const ranks = new Map; + for (const claim of claims) + ranks.set(claim.field, Math.min(ranks.get(claim.field) ?? 1000, rank(claim))); + const chosen = new Set; + return claims.map((claim) => { + const multiple = ["authors", "tags", "alternate"].includes(claim.field); + const selected = rank(claim) < 1000 && rank(claim) === ranks.get(claim.field) && (multiple || !chosen.has(claim.field)); + if (selected) + chosen.add(claim.field); + return { ...claim, selected, rule: `metadata-precedence-v1:${claim.rule}` }; + }); +} +function parseArticleMetadata(input, redact = cleanText) { + if (!record(input) || input.schemaVersion !== 1 || input.extractorVersion !== "wordcell-metadata-v1" || !Array.isArray(input.claims) || input.claims.length > metadataLimits.claims || !Array.isArray(input.warnings)) + throw new Error("Unsupported or malformed article metadata envelope"); + let bytes = 0; + const text = (value2, limit = metadataLimits.value) => { + if (typeof value2 !== "string" || value2.length > limit) + throw new Error("Invalid article metadata text"); + const result = cleanText(redact(value2)); + bytes += Buffer.byteLength(result); + if (bytes > metadataLimits.totalBytes) + throw new Error("Article metadata exceeds byte limit"); + return result; + }; + const value = (v) => Array.isArray(v) ? v.length <= 32 ? v.map((part) => text(part)) : (() => { + throw new Error("Article metadata list exceeds limit"); + })() : text(v); + const claims = input.claims.map((raw) => { + if (!record(raw) || !sources.has(raw.source) || typeof raw.selected !== "boolean") + throw new Error("Invalid article metadata claim"); + const field = text(raw.field, 128); + if (!/^[a-z][a-z0-9:_-]{0,127}$/.test(field) || credentialName.test(field)) + throw new Error("Invalid article metadata field"); + let observed = value(raw.observed), normalized = value(raw.value); + if (urlFields.has(field)) { + const urls = (input2) => { + const normalize = (part) => { + const parsed = publicUrl(part); + if (parsed === null) + throw new Error("Invalid article metadata URL"); + return parsed; + }; + return typeof input2 === "string" ? normalize(input2) : input2.map(normalize); + }; + observed = urls(observed); + normalized = urls(normalized); + } + const temporal = raw.datePrecision === undefined ? {} : dateFields.has(field) && typeof observed === "string" ? dateValue(observed) : (() => { + throw new Error("Invalid article metadata date"); + })(); + if (raw.datePrecision !== undefined && (raw.datePrecision !== temporal.datePrecision || raw.timezone !== temporal.timezone)) + throw new Error("Invalid article metadata date precision"); + return { field, observed, value: normalized, source: raw.source, locator: text(raw.locator, 512), rule: text(raw.rule, 256), selected: raw.selected, ...temporal }; + }); + if (input.warnings.length > warningKinds.size || input.warnings.some((warning) => !warningKinds.has(warning))) + throw new Error("Invalid article metadata warnings"); + return { schemaVersion: 1, extractorVersion: "wordcell-metadata-v1", claims, warnings: [...input.warnings] }; +} +var fields = { + "og:title": "title", + "twitter:title": "title", + title: "title", + citation_title: "title", + "dc.title": "title", + description: "description", + "og:description": "description", + "twitter:description": "description", + "dc.description": "description", + author: "authors", + "article:author": "authors", + citation_author: "authors", + "dc.creator": "authors", + "article:published_time": "published", + datepublished: "published", + citation_publication_date: "published", + "dc.date": "published", + "article:modified_time": "modified", + datemodified: "modified", + "last-modified": "modified", + "og:url": "canonical", + "twitter:url": "canonical", + "og:image": "image", + "og:image:url": "image", + "twitter:image": "image", + "twitter:image:src": "image", + "og:site_name": "site", + publisher: "publisher", + "dc.publisher": "publisher", + citation_journal_title: "publisher", + "og:locale": "language", + language: "language", + "dc.language": "language", + "og:type": "type", + "twitter:card": "card", + "article:tag": "tags", + keywords: "tags", + news_keywords: "tags", + citation_keywords: "tags", + citation_doi: "doi", + "dc.identifier": "identifier", + citation_pdf_url: "alternate" +}; +async function extractArticleMetadata(html, base, fallback = {}) { + const claims = []; + const warnings = new Set; + let totalBytes = 0, title = "", script = "", scriptIndex = 0, activeScript = false, jsonBytes = 0; + const add = (field, raw, source, locator, decode = false, evidenceRule) => { + if (claims.length >= metadataLimits.claims) { + warnings.add("claim-limit"); + return; + } + let values = Array.isArray(raw) ? raw : [raw]; + if (values.length > 32) { + warnings.add("value-limit"); + values = values.slice(0, 32); + } + const strings = []; + for (const value2 of values) { + if (typeof value2 !== "string") + continue; + if (value2.length > metadataLimits.value) { + warnings.add("value-limit"); + continue; + } + const decoded = decode ? decodeHTML(value2) : value2; + const cleaned = cleanText(urlFields.has(field) ? publicUrl(decoded, base) ?? "" : decoded); + if (cleaned) + strings.push(cleaned); + } + if (!strings.length) + return; + const observed = Array.isArray(raw) ? strings : strings[0]; + let value = observed; + if (urlFields.has(field)) { + const urls = strings.map((part) => publicUrl(part, base)).filter((part) => part !== null); + if (!urls.length) + return; + value = Array.isArray(raw) ? urls : urls[0]; + } + const temporal = dateFields.has(field) && typeof observed === "string" ? dateValue(observed) : {}; + const claim = { field, observed, value, source, locator, rule: `trim-whitespace-url-query-redaction${evidenceRule === undefined ? "" : `:${evidenceRule}`}`, selected: false, ...temporal }; + const bytes = Buffer.byteLength(JSON.stringify(claim)); + if (totalBytes + bytes > metadataLimits.totalBytes / 2) { + warnings.add("claim-limit"); + return; + } + totalBytes += bytes; + claims.push(claim); + }; + const parseLd = (raw, index) => { + if (!raw.trim()) + return; + if (Buffer.byteLength(raw) > metadataLimits.jsonBytes || (jsonBytes += Buffer.byteLength(raw)) > metadataLimits.jsonBytes * 4) { + warnings.add("json-limit"); + return; + } + let parsed; + try { + parsed = JSON.parse(raw); + } catch { + warnings.add("json-invalid"); + return; + } + let nodes = 0; + const stack = [{ value: parsed, depth: 0 }]; + while (stack.length) { + const node = stack.pop(); + if (++nodes > metadataLimits.jsonNodes || node.depth > metadataLimits.jsonDepth) { + warnings.add("json-limit"); + return; + } + const children = Array.isArray(node.value) ? node.value : record(node.value) ? Object.values(node.value) : []; + if (children.length + nodes + stack.length > metadataLimits.jsonNodes) { + warnings.add("json-limit"); + return; + } + for (const child of children) + stack.push({ value: child, depth: node.depth + 1 }); + } + const items = Array.isArray(parsed) ? parsed.map((value, i) => ({ value, path: `$[${i}]` })) : [{ value: parsed, path: "$" }]; + for (let cursor = 0;cursor < items.length && cursor < metadataLimits.jsonNodes; cursor++) { + const item = items[cursor]; + if (!record(item.value)) + continue; + const object = item.value; + if (Array.isArray(object["@graph"])) + items.push(...object["@graph"].map((value, i) => ({ value, path: `${item.path}.@graph[${i}]` }))); + const types = Array.isArray(object["@type"]) ? object["@type"] : [object["@type"]]; + const entityTypes = types.flatMap((type) => { + const match = typeof type === "string" ? /^(?:(?:https?:\/\/)?schema\.org\/)?(Article|NewsArticle|BlogPosting|ScholarlyArticle|TechArticle|Report|WebPage|VideoObject|AudioObject|PodcastEpisode)$/.exec(type) : null; + return match?.[1] === undefined ? [] : [match[1]]; + }); + const entityType = entityTypes.find((type) => type !== "WebPage") ?? entityTypes[0]; + if (entityType === undefined) + continue; + const names = (v) => Array.isArray(v) ? v.slice(0, 32).map((entry) => record(entry) ? entry.name : entry) : record(v) ? v.name : v; + const props = { headline: "title", name: "title", description: "description", author: "authors", creator: "authors", datePublished: "published", dateModified: "modified", inLanguage: "language", publisher: "publisher", keywords: "tags", image: "image", thumbnailUrl: "image", url: "canonical", mainEntityOfPage: "canonical", duration: "duration", episodeNumber: "episode", identifier: "identifier" }; + for (const [key, field] of Object.entries(props)) { + let value = object[key]; + if (value === undefined) + continue; + if (["author", "creator", "publisher"].includes(key)) + value = names(value); + if (key === "image") + value = Array.isArray(value) ? value.map((v) => record(v) ? v.url ?? v.contentUrl : v) : record(value) ? value.url ?? value.contentUrl : value; + if (key === "mainEntityOfPage" && record(value)) + value = value["@id"] ?? value.url; + if (key === "episodeNumber" && typeof value === "number" && Number.isFinite(value)) + value = String(value); + add(field, value, "json-ld", `script[${index}]${item.path}.${key}`, false, `jsonld-type=${entityType}`); + } + } + }; + let input = html; + if (Buffer.byteLength(input) > metadataLimits.htmlBytes) { + input = Buffer.from(input).subarray(0, metadataLimits.htmlBytes).toString("utf8"); + warnings.add("input-limit"); + } + await new HTMLRewriter().on("html", { element(element) { + add("language", element.getAttribute("lang"), "html-language", "html@lang", true); + } }).on("title", { text(chunk) { + if (title.length <= metadataLimits.value) + title += chunk.text; + } }).on("meta", { element(element) { + const attribute = element.getAttribute("property") ? "property" : "name"; + const key = decodeHTML(element.getAttribute(attribute) ?? "").toLowerCase(); + if (!/^[a-z][a-z0-9:._-]{0,100}$/.test(key) || credentialName.test(key)) + return; + const field = Object.hasOwn(fields, key) ? fields[key] : `meta:${key.replaceAll(".", "_")}`; + add(field, element.getAttribute("content"), "html-meta", `meta[${attribute}=${key}]@content`, true); + } }).on("link", { element(element) { + const rel = (element.getAttribute("rel") ?? "").toLowerCase().split(/\s+/); + const field = rel.includes("canonical") ? "canonical" : rel.includes("alternate") ? "alternate" : null; + if (field) + add(field, element.getAttribute("href"), "html-link", `link[rel=${field}]@href`, true); + } }).on("script", { element(element) { + activeScript = (element.getAttribute("type") ?? "").toLowerCase().trim() === "application/ld+json"; + script = ""; + if (activeScript) { + const index = scriptIndex++; + if (index >= 64) { + activeScript = false; + warnings.add("json-limit"); + return; + } + element.onEndTag(() => { + parseLd(script, index); + activeScript = false; + }); + } + }, text(chunk) { + if (activeScript && script.length <= metadataLimits.jsonBytes) + script += chunk.text; + } }).transform(new Response(input)).text(); + if (activeScript) + warnings.add("json-invalid"); + add("title", title, "html-title", "title", true); + for (const [key, field] of Object.entries({ title: "title", author: "authors", published: "published", description: "description" })) + add(field, fallback[key], "extractor", `defuddle.${key}`); + return parseArticleMetadata({ schemaVersion: 1, extractorVersion: "wordcell-metadata-v1", claims: select(claims), warnings: [...warnings] }); +} + +export { parseArticleMetadata, extractArticleMetadata }; diff --git a/dist/index-hacpnaew.js b/dist/index-fd8j527z.js similarity index 99% rename from dist/index-hacpnaew.js rename to dist/index-fd8j527z.js index 15235826..2c7886f7 100644 --- a/dist/index-hacpnaew.js +++ b/dist/index-fd8j527z.js @@ -1,7 +1,7 @@ // @bun import { sanitizeArtifactUrl -} from "./index-mxxxytys.js"; +} from "./index-g63d3937.js"; import { assertSafeNetworkUrl, decodeBytes, diff --git a/dist/index-mxxxytys.js b/dist/index-g63d3937.js similarity index 99% rename from dist/index-mxxxytys.js rename to dist/index-g63d3937.js index 90c5d0de..cbc11cec 100644 --- a/dist/index-mxxxytys.js +++ b/dist/index-g63d3937.js @@ -1,4 +1,7 @@ // @bun +import { + parseArticleMetadata +} from "./index-e2wstc7t.js"; import { sanitizeTerminalText } from "./index-1xxnjn0d.js"; @@ -1070,6 +1073,7 @@ function normalizeManifest(input, hasSourceHtml, document) { return { schemaVersion: CAPTURE_MANIFEST_SCHEMA_VERSION, document, + ...input.metadata === undefined ? {} : { metadata: parseArticleMetadata(input.metadata, redactSensitiveText) }, sourceUrl: sanitizeArtifactUrl(input.sourceUrl), canonicalUrl: sanitizeArtifactUrl(input.canonicalUrl), capturedAt: new Date(capturedAtTime).toISOString(), diff --git a/dist/index-k86wepd8.js b/dist/index-h2w7syy2.js similarity index 99% rename from dist/index-k86wepd8.js rename to dist/index-h2w7syy2.js index a206889f..1c7709e0 100644 --- a/dist/index-k86wepd8.js +++ b/dist/index-h2w7syy2.js @@ -6,7 +6,10 @@ import { resolveRemote, rewriteContentWithStatus, scanImageSources -} from "./index-cd75vky9.js"; +} from "./index-jbtyp6q5.js"; +import { + parseArticleMetadata +} from "./index-e2wstc7t.js"; import { FetchFailure, safeFetch @@ -812,6 +815,7 @@ async function extractPage(acquisition, scope, timeoutMs = 30000) { article = { content: retainDefuddleMedia(restoredContent, response, acquisition.finalUrl), title: boundedMetadata(response.title, articleMetadataLimits.title) ?? boundedMetadata(acquisition.browserTitle, articleMetadataLimits.title), + ...response.metadata === undefined ? {} : { metadata: parseArticleMetadata(response.metadata) }, author: boundedMetadata(response.author, articleMetadataLimits.author), published: boundedMetadata(response.published, articleMetadataLimits.published), description diff --git a/dist/index-cd75vky9.js b/dist/index-jbtyp6q5.js similarity index 100% rename from dist/index-cd75vky9.js rename to dist/index-jbtyp6q5.js diff --git a/dist/index-6c8xhred.js b/dist/index-k9yzc4vn.js similarity index 100% rename from dist/index-6c8xhred.js rename to dist/index-k9yzc4vn.js diff --git a/dist/index-m9p2a4mj.js b/dist/index-my50xsa3.js similarity index 83% rename from dist/index-m9p2a4mj.js rename to dist/index-my50xsa3.js index a0e74b5d..b40caee5 100644 --- a/dist/index-m9p2a4mj.js +++ b/dist/index-my50xsa3.js @@ -2,7 +2,7 @@ import { countWords, sniffImage -} from "./index-k86wepd8.js"; +} from "./index-h2w7syy2.js"; import { renderFailure, sentence, @@ -10,11 +10,11 @@ import { stderrStyle, terminalOutput, yamlString -} from "./index-cd75vky9.js"; +} from "./index-jbtyp6q5.js"; import { redactSensitiveText, sanitizeArtifactUrl -} from "./index-mxxxytys.js"; +} from "./index-g63d3937.js"; import { safeFetch } from "./index-e5fbsywq.js"; @@ -26,6 +26,16 @@ import { sanitizeTerminalText } from "./index-1xxnjn0d.js"; +// src/pdf/errors.ts +class PdfCaptureError extends Error { + code; + constructor(code, message, options) { + super(message, options); + this.code = code; + this.name = "PdfCaptureError"; + } +} + // src/pdf/args.ts var pdfUsage = `wordcell pdf \u2014 save a local or public remote PDF as an auditable Markdown bundle @@ -521,7 +531,7 @@ function layoutBlocks(pages) { const repeatedMargins = repeatedMarginLines(pages, linesByPage); const output = []; for (const page of pages) { - const lines = (linesByPage.get(page.page) ?? []).filter((line) => !repeatedMargins.has(normalizedMarginText(line.text))); + const lines = (linesByPage.get(page.page) ?? []).filter((line) => !(line.top <= page.height * 0.06 || line.top + line.height >= page.height * 0.94) || !repeatedMargins.has(normalizedMarginText(line.text))); const steps = lines.slice(1).map((line, index) => Math.max(0, line.top - (lines[index]?.top ?? line.top))); const normalStep = median(steps.filter((step) => step > 0), Math.max(1, bodySize * 1.4)); const events = [ @@ -548,7 +558,7 @@ function layoutBlocks(pages) { const { line, index } = event; const previous = lines[index - 1]; const gapBefore = previous === undefined ? Number.POSITIVE_INFINITY : line.top - previous.top; - const fontHeading = line.fontSize > bodySize * 1.08; + const fontHeading = line.fontSize > bodySize * 1.08 && !/[=\u222B\u2211\u221A\u2070\u00B9\u00B2\u00B3\u2074\u2075\u2076\u2077\u2078\u2079]/u.test(line.text); const boldHeading = line.boldRatio >= 0.78 && line.text.length <= 180 && gapBefore >= normalStep * 1.45; if (line.text.length <= 240 && (fontHeading || boldHeading)) { flushParagraph(); @@ -572,7 +582,7 @@ function layoutBlocks(pages) { } flushParagraph(); } - return output.filter((block) => block.kind === "image" || block.text.trim() !== ""); + return output.filter((block) => block.kind === "image" || block.kind === "table" || block.text.trim() !== ""); } function parsePdfImageInterpretations(value) { if (!Array.isArray(value)) @@ -673,6 +683,160 @@ function parsePdfImageInterpretations(value) { } return output; } +function analyzePdfLayout(pages) { + const issues = []; + const blocks = []; + const marginLines = new Map(pages.filter((page) => page.text.length <= 4000).map((page) => [page.page, pageLines(page)])); + const margins = repeatedMarginLines(pages, marginLines); + for (const originalInput of pages) { + const original = { ...originalInput, text: originalInput.text.filter((fragment) => !(fragment.top <= originalInput.height * 0.06 || fragment.top + fragment.height >= originalInput.height * 0.94) || !margins.has(normalizedMarginText(fragment.text))) }; + const warn = (code) => { + if (!issues.some((issue) => issue.page === original.page && issue.code === code)) + issues.push({ page: original.page, code }); + }; + if (original.text.length > 4000) { + warn("reading-order-uncertain"); + blocks.push(...layoutBlocks([original])); + continue; + } + if (original.text.some((fragment) => /[\u222B\u2211\u221A]/u.test(fragment.text))) + warn("math-structure-uncertain"); + const text = [...original.text].sort((a, b) => a.top - b.top || a.left - b.left); + const consumed = new Set; + const replacements = new Map; + const superscripts = { "0": "\u2070", "1": "\xB9", "2": "\xB2", "3": "\xB3", "4": "\u2074", "5": "\u2075", "6": "\u2076", "7": "\u2077", "8": "\u2078", "9": "\u2079", "+": "\u207A", "-": "\u207B", "=": "\u207C", "(": "\u207D", ")": "\u207E", n: "\u207F", i: "\u2071" }; + for (const small of text) { + const bases = text.filter((base2) => base2 !== small && base2.fontSize >= small.fontSize * 1.3 && small.top < base2.top && small.top + small.height > base2.top - base2.height * 0.25 && Math.abs(small.left - (base2.left + base2.width)) < base2.fontSize * 0.6); + if (bases.length === 0) + continue; + const base = bases[0]; + const glyphs = [...small.text.trim()]; + if (bases.length === 1 && glyphs.length > 0 && glyphs.length <= 8 && glyphs.every((glyph) => superscripts[glyph] !== undefined) && !replacements.has(base)) { + replacements.set(base, { ...base, text: base.text + glyphs.map((glyph) => superscripts[glyph]).join(""), width: small.left + small.width - base.left }); + consumed.add(small); + } else + warn("math-structure-uncertain"); + } + const fragments = text.filter((fragment) => !consumed.has(fragment)).map((fragment) => replacements.get(fragment) ?? fragment); + const page = { ...original, text: fragments }; + const rows = []; + for (const fragment of fragments) { + const row = rows.at(-1); + if (row === undefined || Math.abs(fragment.top - row[0].top) > Math.max(2, fragment.height * 0.2)) + rows.push([fragment]); + else + row.push(fragment); + } + const cellRows = rows.map((row) => { + const groups = []; + for (const fragment of [...row].sort((a, b) => a.left - b.left)) { + const prior = groups.at(-1), last = prior?.at(-1); + if (last === undefined || fragment.left - (last.left + last.width) > Math.max(12, fragment.fontSize * 1.5)) + groups.push([fragment]); + else + prior.push(fragment); + } + return groups; + }); + const tables = []; + for (let index = 0;index < cellRows.length; index++) { + const first = cellRows[index]; + if (first.length < 2 || first.length > 12) + continue; + const group = [first]; + let next = index + 1; + for (;next < cellRows.length; next++) { + const candidate = cellRows[next]; + if (candidate.length !== first.length || candidate.some((cell, column) => Math.abs(cell[0].left - first[column][0].left) > 4)) + break; + if (candidate[0][0].top - group.at(-1)[0][0].top > Math.max(40, candidate[0][0].height * 3)) + break; + group.push(candidate); + } + const cells = group.map((row) => row.map((cell) => joinFragments(cell))); + const header = first.every((cell) => cell.every((fragment) => fragment.bold)); + const numeric = cells.slice(1).some((row) => row.slice(1).some((cell) => /^[-+\d$\u20AC\u00A3]/u.test(cell.trim()))); + const compact = cells.every((row) => row.every((cell) => cell.length <= 100)); + if (group.length >= 3 && compact && header && numeric) { + const members = group.flat(2); + tables.push({ top: Math.min(...members.map((fragment) => fragment.top)), bottom: Math.max(...members.map((fragment) => fragment.top + fragment.height)), block: { kind: "table", page: page.page, rows: cells }, fragments: members }); + index = next - 1; + } else if (group.length >= 3 && first.length >= 3) + warn("table-structure-uncertain"); + } + const tableFragments = new Set(tables.flatMap((table) => table.fragments)); + const remaining = fragments.filter((fragment) => !tableFragments.has(fragment)); + const emitBand = (band, images) => { + if (band.length === 0 && images.length === 0) + return; + const candidates = new Set; + for (const row of cellRows) { + for (let i = 1;i < row.length; i++) { + const left = row[i - 1].at(-1), right = row[i][0]; + const gap = right.left - (left.left + left.width); + if (gap >= Math.max(page.width * 0.025, left.fontSize * 1.5)) + candidates.add(Math.round((left.left + left.width + right.left) / 2)); + } + } + const valid = [...candidates].filter((gutter) => { + const left = band.filter((fragment) => fragment.left + fragment.width <= gutter); + const right = band.filter((fragment) => fragment.left >= gutter); + if (left.length + right.length !== band.length || left.length < 3 || right.length < 3) + return false; + if (left.filter((fragment) => fragment.width > page.width * 0.18).length < 2 || right.filter((fragment) => fragment.width > page.width * 0.18).length < 2) + return false; + const overlap = Math.min(Math.max(...left.map((fragment) => fragment.top)), Math.max(...right.map((fragment) => fragment.top))) - Math.max(Math.min(...left.map((fragment) => fragment.top)), Math.min(...right.map((fragment) => fragment.top))); + return overlap > Math.max(...band.map((fragment) => fragment.height)) * 1.5; + }); + if (valid.length > 0) { + const gutter = valid[Math.floor(valid.length / 2)]; + if (images.some((image) => image.left < gutter && image.left + image.width > gutter)) { + warn("reading-order-uncertain"); + blocks.push(...layoutBlocks([{ ...page, text: band, images }])); + return; + } + for (const left of [true, false]) + blocks.push(...layoutBlocks([{ ...page, text: band.filter((fragment) => left ? fragment.left + fragment.width <= gutter : fragment.left >= gutter), images: images.filter((image) => left ? image.left + image.width <= gutter : image.left >= gutter) }])); + } else { + if (band.length >= 6 && cellRows.filter((row) => row.length >= 2 && row.some((cell) => band.includes(cell[0]))).length >= 3) + warn("reading-order-uncertain"); + blocks.push(...layoutBlocks([{ ...page, text: band, images }])); + } + }; + const bodyFont = weightedBodyFontSize(pageLines(page)); + const gutterVotes = new Map; + for (const row of cellRows) + for (let i = 1;i < row.length; i++) { + const left = row[i - 1].at(-1), right = row[i][0]; + if (right.left - (left.left + left.width) < Math.max(12, left.fontSize * 1.5)) + continue; + const gutter = Math.round((left.left + left.width + right.left) / 20) * 10; + gutterVotes.set(gutter, (gutterVotes.get(gutter) ?? 0) + 1); + } + const repeatedGutters = [...gutterVotes].filter(([, votes]) => votes >= 3).map(([gutter]) => gutter); + const spans = remaining.filter((fragment) => fragment.width > page.width * 0.65 || fragment.fontSize > bodyFont * 1.08 && repeatedGutters.some((gutter) => fragment.left < gutter && fragment.left + fragment.width > gutter)); + const boundaries = [...tables.map((table) => ({ top: table.top, bottom: table.bottom, table, span: null })), ...spans.map((span) => ({ top: span.top, bottom: span.top + span.height, table: null, span }))].sort((a, b) => a.top - b.top); + const emitted = new Set, emittedImages = new Set; + for (const boundary of boundaries) { + const band = remaining.filter((fragment) => !emitted.has(fragment) && fragment !== boundary.span && fragment.top < boundary.top); + const images = page.images.filter((image) => !emittedImages.has(image) && image.top < boundary.top); + emitBand(band, images); + for (const fragment of band) + emitted.add(fragment); + for (const image of images) + emittedImages.add(image); + if (boundary.table !== null) + blocks.push(boundary.table.block); + if (boundary.span !== null) { + const span = boundary.span; + blocks.push(span.fontSize > bodyFont * 1.08 ? { kind: "heading", page: page.page, level: 2, text: span.text } : { kind: "paragraph", page: page.page, text: span.text }); + emitted.add(span); + } + } + emitBand(remaining.filter((fragment) => !emitted.has(fragment)), page.images.filter((image) => !emittedImages.has(image))); + } + return { blocks, issues }; +} // src/pdf/tools.ts import { spawn } from "child_process"; @@ -696,7 +860,7 @@ async function readBoundedStream(stream, maxBytes) { else throw new Error("PDF tool returned an unsupported output chunk"); if (!bytes.append(chunk)) - throw new Error(`PDF tool output exceeded ${maxBytes} bytes`); + throw new PdfCaptureError("limit-exceeded", `PDF tool output exceeded ${maxBytes} bytes`); } return new TextDecoder().decode(bytes.toUint8Array()); } @@ -704,7 +868,9 @@ var runPdfToolCommand = async (specification) => { const executable = specification.command[0]; if (executable === undefined) throw new Error("PDF tool command is empty"); - const useProcessGroup = process.platform !== "win32"; + if (specification.processGroup !== undefined && !["new", "inherit"].includes(specification.processGroup)) + throw new PdfCaptureError("invalid-input", "Invalid PDF process group policy"); + const useProcessGroup = process.platform !== "win32" && specification.processGroup !== "inherit"; const child = spawn(executable, specification.command.slice(1), { cwd: specification.cwd, detached: useProcessGroup, @@ -718,7 +884,7 @@ var runPdfToolCommand = async (specification) => { windowsHide: true }); const exited = new Promise((resolveExit, rejectExit) => { - child.once("error", rejectExit); + child.once("error", (error) => rejectExit(error.code === "ENOENT" ? new PdfCaptureError("tool-unavailable", "PDF tool executable is unavailable", { cause: error }) : error)); child.once("close", (code) => resolveExit(code ?? 1)); }); const signalProcessTree = (signal) => { @@ -742,7 +908,7 @@ var runPdfToolCommand = async (specification) => { forceKillTimer = setTimeout(() => signalProcessTree("SIGKILL"), 1000); }; const timer = setTimeout(() => { - requestStop(new Error(`PDF tool timed out after ${specification.timeoutMs}ms`)); + requestStop(new PdfCaptureError("timeout", `PDF tool timed out after ${specification.timeoutMs}ms`)); }, specification.timeoutMs); try { const [stdout, stderr, exitCode] = await Promise.all([ @@ -791,10 +957,10 @@ function resolvePdfTools(dependencies = {}) { const pdftohtml = dependencies.tools?.pdftohtml ?? discoverExecutable("pdftohtml", dependencies); const tesseract = dependencies.tools?.tesseract === undefined ? discoverExecutable("tesseract", dependencies) : dependencies.tools.tesseract; if (pdfinfo === null) { - throw new Error("pdfinfo is required for PDF ingestion; install the Poppler command-line tools"); + throw new PdfCaptureError("tool-unavailable", "pdfinfo is required for PDF ingestion; install the Poppler command-line tools"); } if (pdftohtml === null) { - throw new Error("pdftohtml is required for PDF ingestion; install the Poppler command-line tools"); + throw new PdfCaptureError("tool-unavailable", "pdftohtml is required for PDF ingestion; install the Poppler command-line tools"); } return { pdfinfo, pdftohtml, tesseract }; } @@ -828,7 +994,7 @@ var pdfCaptureDefaults = { function positiveBound(value, fallback, maximum, label) { const selected = value ?? fallback; if (!Number.isSafeInteger(selected) || selected < 1 || selected > maximum) { - throw new Error(`${label} must be an integer from 1 through ${maximum}`); + throw new PdfCaptureError("invalid-input", `${label} must be an integer from 1 through ${maximum}`); } return selected; } @@ -841,10 +1007,10 @@ function prepareWorkspace(path) { if (existsSync2(absolute)) { const stats = lstatSync(absolute); if (!stats.isDirectory() || stats.isSymbolicLink()) { - throw new Error("PDF inspection workspace must be a real directory"); + throw new PdfCaptureError("invalid-input", "PDF inspection workspace must be a real directory"); } if (readdirSync(absolute).length !== 0) { - throw new Error("PDF inspection workspace must be empty"); + throw new PdfCaptureError("invalid-input", "PDF inspection workspace must be empty"); } } else { mkdirSync(absolute, { recursive: false, mode: 448 }); @@ -857,17 +1023,17 @@ function sourceIdentity(path, maxPdfBytes) { const canonical = realpathSync(resolve(path)); const stats = statSync(canonical); if (!stats.isFile()) - throw new Error("PDF input must be a regular file"); + throw new PdfCaptureError("invalid-input", "PDF input must be a regular file"); if (stats.size < 5) - throw new Error("PDF input is too small to contain a PDF header"); + throw new PdfCaptureError("invalid-input", "PDF input is too small to contain a PDF header"); if (stats.size > maxPdfBytes) - throw new Error(`PDF input exceeds the ${maxPdfBytes}-byte limit`); + throw new PdfCaptureError("limit-exceeded", `PDF input exceeds the ${maxPdfBytes}-byte limit`); const descriptor = openSync(canonical, "r"); try { const signature = Buffer.alloc(5); const count = readSync(descriptor, signature, 0, signature.length, 0); if (count !== signature.length || signature.toString("ascii") !== "%PDF-") { - throw new Error("PDF input does not have a valid PDF signature"); + throw new PdfCaptureError("invalid-input", "PDF input does not have a valid PDF signature"); } } finally { closeSync(descriptor); @@ -923,18 +1089,18 @@ function safeGeneratedImage(image, workspace, index, limits) { function generatedTreeSize(workspace, maximumFiles, maximumBytes) { const entries = readdirSync(workspace, { withFileTypes: true }); if (entries.length > maximumFiles) { - throw new Error(`PDF extraction created more than ${maximumFiles} files`); + throw new PdfCaptureError("limit-exceeded", `PDF extraction created more than ${maximumFiles} files`); } let totalBytes = 0; for (const entry of entries) { const path = join2(workspace, entry.name); const stats = lstatSync(path); if (!entry.isFile() || stats.isSymbolicLink()) { - throw new Error("PDF extraction created an unexpected non-file output"); + throw new PdfCaptureError("extraction-failed", "PDF extraction created an unexpected non-file output"); } totalBytes += stats.size; if (totalBytes > maximumBytes) { - throw new Error(`PDF extraction exceeded the ${maximumBytes}-byte workspace limit`); + throw new PdfCaptureError("limit-exceeded", `PDF extraction exceeded the ${maximumBytes}-byte workspace limit`); } } } @@ -957,7 +1123,8 @@ async function inspectPdf(options, dependencies = {}) { cwd: workspaceDirectory }); if (info.exitCode !== 0) { - throw new Error("pdfinfo could not inspect the input PDF"); + const encrypted = info.stderr.split(/\r?\n/u).some((line) => line.trim() === "Command Line Error: Incorrect password"); + throw new PdfCaptureError(encrypted ? "encrypted" : "extraction-failed", "pdfinfo could not inspect the input PDF"); } const metadata = parsePdfInfo(info.stdout); const processedPageLimit = Math.min(metadata.pageCount, maxPages); @@ -982,12 +1149,12 @@ async function inspectPdf(options, dependencies = {}) { cwd: workspaceDirectory }); if (extracted.exitCode !== 0) { - throw new Error("pdftohtml could not extract the input PDF"); + throw new PdfCaptureError("extraction-failed", "pdftohtml could not extract the input PDF"); } generatedTreeSize(workspaceDirectory, 10002, Math.min(Number.MAX_SAFE_INTEGER, 8 * 1024 ** 3 + pdfCaptureDefaults.maxLayoutBytes)); const layoutStats = statSync(layoutPath); if (!layoutStats.isFile() || layoutStats.size > pdfCaptureDefaults.maxLayoutBytes) { - throw new Error(`Poppler layout XML exceeds the ${pdfCaptureDefaults.maxLayoutBytes}-byte limit`); + throw new PdfCaptureError("limit-exceeded", `Poppler layout XML exceeds the ${pdfCaptureDefaults.maxLayoutBytes}-byte limit`); } const parsed = parsePopplerXml(readFileSync(layoutPath, "utf8"), { maxPages: processedPageLimit, @@ -1214,8 +1381,14 @@ function persistPdfCapture(input) { copyFileSync(image.sourcePath, destination); chmodSync2(destination, 420); } - writeFileSync(join3(stagingDirectory, `${slug}.md`), `${redactSensitiveText(input.markdown).trimEnd()} -`, { encoding: "utf8", flag: "wx", mode: 420 }); + const markdownBytes = `${redactSensitiveText(input.markdown).trimEnd()} +`; + const markdownDescriptor = { path: `${slug}.md`, bytes: Buffer.byteLength(markdownBytes), sha256: createHash2("sha256").update(markdownBytes).digest("hex") }; + if (input.manifest.markdown !== undefined && (input.manifest.markdown.path !== markdownDescriptor.path || input.manifest.markdown.bytes !== markdownDescriptor.bytes || input.manifest.markdown.sha256 !== markdownDescriptor.sha256)) { + throw new Error("PDF Markdown does not match its manifest provenance"); + } + const manifest = { ...input.manifest, markdown: markdownDescriptor }; + writeFileSync(join3(stagingDirectory, `${slug}.md`), markdownBytes, { encoding: "utf8", flag: "wx", mode: 420 }); if (input.manifest.annotations === null) { if (input.annotationsJson !== undefined) { throw new Error("PDF annotations were supplied without manifest provenance"); @@ -1239,7 +1412,7 @@ function persistPdfCapture(input) { } writeFileSync(join3(stagingDirectory, PDF_CAPTURE_ANNOTATIONS_FILENAME), annotationsJson, { encoding: "utf8", flag: "wx", mode: 420 }); } - writeFileSync(join3(stagingDirectory, PDF_CAPTURE_MANIFEST_FILENAME), `${JSON.stringify(input.manifest, null, 2)} + writeFileSync(join3(stagingDirectory, PDF_CAPTURE_MANIFEST_FILENAME), `${JSON.stringify(manifest, null, 2)} `, { encoding: "utf8", flag: "wx", mode: 420 }); assertStagingTreeSafe(stagingDirectory); if (existsSync3(targetDirectory) !== (expectedIdentity !== null)) { @@ -1297,7 +1470,7 @@ function pdfMarkdownFilename(slug) { // src/pdf/markdown.ts function escapeInline(value) { - return sanitizeTerminalText(value).replace(/\\/gu, "\\\\").replace(/([`*_[\]{}<>#])/gu, "\\$1").replace(/\s+/gu, " ").trim(); + return sanitizeTerminalText(value).replace(/\\/gu, "\\\\").replace(/([`*_[\]{}<>#|])/gu, "\\$1").replace(/\s+/gu, " ").trim(); } function metadataLine(metadata) { if (metadata === null) @@ -1347,6 +1520,13 @@ function markdownContent(blocks, images) { lines.push(imageMarkdown(resolved), ""); continue; } + if (block.kind === "table") { + textBlockCount += 1; + const rows = block.rows.map((row) => `| ${row.map((cell) => escapeInline(cell)).join(" | ")} |`); + if (rows.length > 0) + lines.push(rows[0], `| ${block.rows[0].map(() => "---").join(" | ")} |`, ...rows.slice(1), ""); + continue; + } if (block.kind === "heading") { headingCount += 1; lines.push(`${"#".repeat(Math.max(2, Math.min(6, block.level)))} ${escapeInline(block.text)}`, ""); @@ -1662,13 +1842,16 @@ async function runPdfCapture(options, dependencies = {}) { }); } const platforms = embeddedPlatforms(resolvedImages); + const layout = analyzePdfLayout(inspection.pages); + for (const issue of layout.issues) + warnings.push(`Page ${issue.page}: ${issue.code}; consult the retained source PDF.`); const status = warnings.length > 0 || unclassifiedImages > 0 || ocrWarningCount > 0 ? "partial" : "complete"; const filenameTitle = inspection.originalFilename.replace(/\.pdf$/iu, "").trim(); const requestedSlug = options.slug ?? inspection.metadata.title ?? filenameTitle; const slug = slugify(sanitizeTerminalText(requestedSlug)); if (slug === "") throw new Error("could not derive a safe PDF capture slug; pass an explicit slug"); - const blocks = layoutBlocks(inspection.pages); + const blocks = layout.blocks; const byId = new Map(resolvedImages.map((image) => [image.image.id, image])); const capturedAt = (dependencies.now ?? (() => new Date))().toISOString(); const built = buildPdfMarkdown({ @@ -1701,6 +1884,8 @@ async function runPdfCapture(options, dependencies = {}) { const manifest = { schemaVersion: PDF_CAPTURE_MANIFEST_SCHEMA_VERSION, kind: "pdf", + quality: { schemaVersion: 1, issues: layout.issues }, + markdown: { path: pdfMarkdownFilename(slug), bytes: Buffer.byteLength(built.markdown), sha256: createHash3("sha256").update(built.markdown).digest("hex") }, capturedAt, status, source: { @@ -1785,13 +1970,13 @@ function parseRemoteUrl(input) { try { url = new URL(input); } catch { - throw new Error("PDF URL is invalid"); + throw new PdfCaptureError("invalid-input", "PDF URL is invalid"); } if (url.protocol !== "http:" && url.protocol !== "https:") { - throw new Error("PDF URL must use HTTP or HTTPS"); + throw new PdfCaptureError("invalid-input", "PDF URL must use HTTP or HTTPS"); } if (url.username !== "" || url.password !== "") { - throw new Error("PDF URL must not contain embedded credentials"); + throw new PdfCaptureError("invalid-input", "PDF URL must not contain embedded credentials"); } return url; } @@ -1808,7 +1993,7 @@ function remoteFilename(url) { } function assertPdfSignature(bytes) { if (bytes.byteLength < 5 || new TextDecoder().decode(bytes.subarray(0, 5)) !== "%PDF-") { - throw new Error("remote PDF input does not have a valid PDF signature"); + throw new PdfCaptureError("invalid-input", "remote PDF input does not have a valid PDF signature"); } } async function preparePdfSource(input, options = {}, dependencies = {}) { @@ -1961,7 +2146,7 @@ async function main(rawArguments = process.argv.slice(2), environment = process. } catch (error) { const message = safe(error instanceof Error ? error.message : String(error)); if (arguments_.json) - output.stdout(terminalSafeJson({ ok: false, error: message })); + output.stdout(terminalSafeJson({ ok: false, error: message, code: error instanceof PdfCaptureError ? error.code : "unknown" })); else output.stderr(renderFailure(message, "wordcell doctor", stderrStyle(environment, output))); return 1; @@ -1972,4 +2157,4 @@ async function main(rawArguments = process.argv.slice(2), environment = process. if (false) ; -export { pdfUsage, parsePdfArguments, decodePopplerText, parsePopplerXml, parsePdfInfo, layoutBlocks, parsePdfImageInterpretations, runPdfToolCommand, resolvePdfTools, pdfCaptureDefaults, inspectPdf, PDF_CAPTURE_MANIFEST_SCHEMA_VERSION, PDF_CAPTURE_MANIFEST_FILENAME, PDF_CAPTURE_SOURCE_FILENAME, PDF_CAPTURE_ANNOTATIONS_FILENAME, pdfImageAssetPath, persistPdfCapture, pdfMarkdownFilename, buildPdfMarkdown, parseTesseractTsv, ocrPdfImage, runPdfCapture, pdfCaptureSummary, main }; +export { PdfCaptureError, pdfUsage, parsePdfArguments, decodePopplerText, parsePopplerXml, parsePdfInfo, layoutBlocks, parsePdfImageInterpretations, runPdfToolCommand, resolvePdfTools, pdfCaptureDefaults, inspectPdf, PDF_CAPTURE_MANIFEST_SCHEMA_VERSION, PDF_CAPTURE_MANIFEST_FILENAME, PDF_CAPTURE_SOURCE_FILENAME, PDF_CAPTURE_ANNOTATIONS_FILENAME, pdfImageAssetPath, persistPdfCapture, pdfMarkdownFilename, buildPdfMarkdown, parseTesseractTsv, ocrPdfImage, runPdfCapture, pdfCaptureSummary, main }; diff --git a/dist/index-ncwzmsge.js b/dist/index-nwdzhj1h.js similarity index 99% rename from dist/index-ncwzmsge.js rename to dist/index-nwdzhj1h.js index 9996c145..a55cb69e 100644 --- a/dist/index-ncwzmsge.js +++ b/dist/index-nwdzhj1h.js @@ -2,7 +2,7 @@ import { classifyPlatformUrl, symbol -} from "./index-cd75vky9.js"; +} from "./index-jbtyp6q5.js"; import { filterCookieProviderResult, readCookieFile, @@ -10,7 +10,7 @@ import { } from "./index-2gv8y733.js"; import { sanitizeArtifactUrl -} from "./index-mxxxytys.js"; +} from "./index-g63d3937.js"; import { startNetworkProxy } from "./index-w2zc0vwa.js"; diff --git a/dist/index-nd8v7r0z.js b/dist/index-sp43012q.js similarity index 99% rename from dist/index-nd8v7r0z.js rename to dist/index-sp43012q.js index 1331aefe..42cb37d2 100644 --- a/dist/index-nd8v7r0z.js +++ b/dist/index-sp43012q.js @@ -1,7 +1,7 @@ // @bun import { isolatedAgentBrowserEnvironment -} from "./index-ncwzmsge.js"; +} from "./index-nwdzhj1h.js"; import { BoundedByteBuffer } from "./index-gh719d91.js"; diff --git a/dist/index-npg9z1a4.js b/dist/index-xn5q4kne.js similarity index 99% rename from dist/index-npg9z1a4.js rename to dist/index-xn5q4kne.js index eaed8ba6..af2e63a9 100644 --- a/dist/index-npg9z1a4.js +++ b/dist/index-xn5q4kne.js @@ -2,7 +2,10 @@ import { CAPTURE_MANIFEST_FILENAME, CAPTURE_SOURCE_EVIDENCE_PATH -} from "./index-mxxxytys.js"; +} from "./index-g63d3937.js"; +import { + parseArticleMetadata +} from "./index-e2wstc7t.js"; // src/clip/bundle-reader.ts import { createHash } from "crypto"; @@ -384,6 +387,7 @@ async function readCaptureBundle(path, options = {}) { } await assertBundleRoot(root); return Object.freeze({ + ...manifest.metadata === undefined ? {} : { metadata: parseArticleMetadata(manifest.metadata) }, root: root.path, schemaVersion, sourceUrl: manifest.sourceUrl, diff --git a/dist/pdf.js b/dist/pdf.js index a8428920..8a20b337 100644 --- a/dist/pdf.js +++ b/dist/pdf.js @@ -4,6 +4,7 @@ import { PDF_CAPTURE_MANIFEST_FILENAME, PDF_CAPTURE_MANIFEST_SCHEMA_VERSION, PDF_CAPTURE_SOURCE_FILENAME, + PdfCaptureError, buildPdfMarkdown, decodePopplerText, inspectPdf, @@ -24,15 +25,442 @@ import { resolvePdfTools, runPdfCapture, runPdfToolCommand -} from "./index-m9p2a4mj.js"; -import"./index-k86wepd8.js"; -import"./index-cd75vky9.js"; -import"./index-mxxxytys.js"; +} from "./index-my50xsa3.js"; +import"./index-h2w7syy2.js"; +import"./index-jbtyp6q5.js"; +import"./index-g63d3937.js"; +import"./index-e2wstc7t.js"; import"./index-e5fbsywq.js"; import"./index-gh719d91.js"; import"./index-1xxnjn0d.js"; import"./index-z1w83f81.js"; +// src/pdf/bundle-reader.ts +import { createHash } from "crypto"; +import { constants as fsConstants } from "fs"; +import { lstat, open, opendir, realpath } from "fs/promises"; +import { dirname, isAbsolute, relative, resolve, sep } from "path"; +function boundedInteger(value, fallback, maximum, label) { + const selected = value ?? fallback; + if (!Number.isSafeInteger(selected) || selected < 1 || selected > maximum) { + throw new PdfCaptureError("invalid-input", `${label} must be an integer from 1 through ${maximum}`); + } + return selected; +} +function confinedRelativePath(value, label) { + if (typeof value !== "string" || value.length === 0 || value.length > 4096 || value.includes("\x00")) { + throw new PdfCaptureError("integrity", `${label} must be a bounded non-empty path`); + } + if (isAbsolute(value) || value.includes("\\")) + throw new PdfCaptureError("integrity", `${label} must be a relative POSIX path`); + const segments = value.split("/"); + if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) { + throw new PdfCaptureError("integrity", `${label} contains an unsafe segment`); + } + return value; +} + +class MissingCaptureAssetError extends PdfCaptureError { + constructor(message) { + super("integrity", message); + } +} +function isMissingPathError(error) { + return error instanceof Error && "code" in error && error.code === "ENOENT"; +} +function assertAssetVerificationDeadline(deadline) { + if (deadline !== undefined && performance.now() >= deadline) { + throw new PdfCaptureError("timeout", "capture asset verification exceeded its time budget"); + } +} +function confined(root, relativePath, label) { + const path = resolve(root.path, relativePath); + const lexical = relative(root.path, path); + if (lexical === "" || lexical === ".." || lexical.startsWith(`..${sep}`) || isAbsolute(lexical)) { + throw new PdfCaptureError("integrity", `${label} escapes the capture bundle`); + } + return path; +} +async function safeDirectory(pathInput) { + if (pathInput.trim() === "") + throw new PdfCaptureError("integrity", "capture bundle path is required"); + const path = resolve(pathInput); + const canonical = await realpath(path); + if (canonical !== path) + throw new PdfCaptureError("integrity", "capture bundle path must be canonical and cannot traverse a symbolic link"); + const stats = await lstat(path, { bigint: true }); + if (!stats.isDirectory() || stats.isSymbolicLink()) + throw new PdfCaptureError("integrity", "capture bundle path must be a real directory"); + if (dirname(path) === path) + throw new PdfCaptureError("integrity", "refusing a filesystem root as a capture bundle"); + const handle = await open(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + try { + const opened = await handle.stat({ bigint: true }); + if (!opened.isDirectory() || opened.dev !== stats.dev || opened.ino !== stats.ino) { + throw new PdfCaptureError("integrity", "capture bundle root changed while it was opened"); + } + return { path, handle, device: opened.dev, inode: opened.ino }; + } catch (error) { + await handle.close(); + throw error; + } +} +async function assertBundleRoot(root, deadline) { + assertAssetVerificationDeadline(deadline); + const [opened, named, canonical] = await Promise.all([ + root.handle.stat({ bigint: true }), + lstat(root.path, { bigint: true }), + realpath(root.path) + ]); + assertAssetVerificationDeadline(deadline); + if (canonical !== root.path || !opened.isDirectory() || !named.isDirectory() || named.isSymbolicLink() || opened.dev !== root.device || opened.ino !== root.inode || named.dev !== root.device || named.ino !== root.inode) + throw new PdfCaptureError("integrity", "capture bundle root changed during inspection"); +} +async function assertAncestorChain(root, relativePath, label, deadline) { + await assertBundleRoot(root, deadline); + const segments = confinedRelativePath(relativePath, label).split("/"); + let current = root.path; + for (const segment of segments.slice(0, -1)) { + current = resolve(current, segment); + assertAssetVerificationDeadline(deadline); + const metadata = await lstat(current, { bigint: true }); + assertAssetVerificationDeadline(deadline); + if (!metadata.isDirectory() || metadata.isSymbolicLink()) { + throw new PdfCaptureError("integrity", `${label} must not traverse a linked or non-directory ancestor`); + } + const canonical = await realpath(current); + assertAssetVerificationDeadline(deadline); + if (canonical !== current) { + throw new PdfCaptureError("integrity", `${label} must not traverse a filesystem alias`); + } + } + await assertBundleRoot(root, deadline); +} +async function withRegularFile(root, relativePath, maximumBytes, label, read, deadline, missingError) { + await assertAncestorChain(root, relativePath, label, deadline); + const path = confined(root, relativePath, label); + assertAssetVerificationDeadline(deadline); + let before; + try { + before = await lstat(path, { bigint: true }); + } catch (error) { + assertAssetVerificationDeadline(deadline); + if (missingError !== undefined && isMissingPathError(error)) + throw missingError; + throw error; + } + assertAssetVerificationDeadline(deadline); + if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1n) { + throw new PdfCaptureError("integrity", `${label} must be a regular single-link file`); + } + if (before.size > BigInt(maximumBytes)) + throw new PdfCaptureError("integrity", `${label} exceeds ${maximumBytes} bytes`); + const canonical = await realpath(path); + assertAssetVerificationDeadline(deadline); + if (canonical !== path) + throw new PdfCaptureError("integrity", `${label} must not traverse a filesystem alias`); + const handle = await open(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + try { + assertAssetVerificationDeadline(deadline); + const opened = await handle.stat({ bigint: true }); + assertAssetVerificationDeadline(deadline); + if (!opened.isFile() || opened.nlink !== 1n || opened.dev !== before.dev || opened.ino !== before.ino || opened.size !== before.size) + throw new PdfCaptureError("integrity", `${label} changed while it was opened`); + await assertAncestorChain(root, relativePath, label, deadline); + const result = await read(handle, Number(opened.size)); + assertAssetVerificationDeadline(deadline); + const after = await handle.stat({ bigint: true }); + assertAssetVerificationDeadline(deadline); + const named = await lstat(path, { bigint: true }); + assertAssetVerificationDeadline(deadline); + await assertAncestorChain(root, relativePath, label, deadline); + const finalCanonical = await realpath(path); + assertAssetVerificationDeadline(deadline); + if (after.dev !== before.dev || after.ino !== before.ino || after.size !== before.size || after.mtimeNs !== before.mtimeNs || after.nlink !== 1n || named.dev !== before.dev || named.ino !== before.ino || named.size !== before.size || named.mtimeNs !== before.mtimeNs || named.nlink !== 1n || named.isSymbolicLink() || finalCanonical !== path) + throw new PdfCaptureError("integrity", `${label} changed while it was read`); + return result; + } finally { + await handle.close(); + assertAssetVerificationDeadline(deadline); + } +} +async function readRegularFile(root, relativePath, maximumBytes, label, deadline) { + return withRegularFile(root, relativePath, maximumBytes, label, async (handle, size) => { + const bytes = Buffer.alloc(size); + let offset = 0; + while (offset < bytes.byteLength) { + assertAssetVerificationDeadline(deadline); + const result = await handle.read(bytes, offset, bytes.byteLength - offset, offset); + if (result.bytesRead === 0) + throw new PdfCaptureError("integrity", `${label} changed while it was read`); + offset += result.bytesRead; + } + return bytes; + }, deadline); +} +async function hashRegularFile(root, relativePath, maximumBytes, deadline, label) { + assertAssetVerificationDeadline(deadline); + await assertBundleRoot(root, deadline); + assertAssetVerificationDeadline(deadline); + await assertAncestorChain(root, relativePath, label, deadline); + const path = confined(root, relativePath, label); + assertAssetVerificationDeadline(deadline); + try { + await lstat(path, { bigint: true }); + } catch (error) { + assertAssetVerificationDeadline(deadline); + if (isMissingPathError(error)) { + await assertBundleRoot(root, deadline); + throw new MissingCaptureAssetError(`capture asset ${relativePath} is missing`); + } + throw error; + } + assertAssetVerificationDeadline(deadline); + return withRegularFile(root, relativePath, maximumBytes, label, async (handle, size) => { + assertAssetVerificationDeadline(deadline); + const hash = createHash("sha256"); + const chunk = Buffer.allocUnsafe(Math.min(1 * 1024 * 1024, Math.max(1, size))); + let offset = 0; + while (offset < size) { + assertAssetVerificationDeadline(deadline); + const result = await handle.read(chunk, 0, Math.min(chunk.byteLength, size - offset), offset); + assertAssetVerificationDeadline(deadline); + if (result.bytesRead === 0) + throw new PdfCaptureError("integrity", `${label} changed while it was read`); + hash.update(chunk.subarray(0, result.bytesRead)); + offset += result.bytesRead; + } + assertAssetVerificationDeadline(deadline); + return { bytes: size, sha256: hash.digest("hex") }; + }, deadline, new MissingCaptureAssetError(`capture asset ${relativePath} is missing`)); +} +function digest(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} +function invalid(message) { + throw new PdfCaptureError("invalid-input", message); +} +function object(value) { + if (value === null || typeof value !== "object" || Array.isArray(value)) + invalid("PDF bundle requires an object"); + return value; +} +function integer(value, maximum, minimum = 0) { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < minimum || value > maximum) + invalid("PDF bundle has an invalid count or page reference"); + return value; +} +function descriptor(value) { + const entry = object(value); + if (typeof entry.sha256 !== "string" || !/^[a-f0-9]{64}$/u.test(entry.sha256)) + invalid("PDF bundle has an invalid digest"); + return { path: confinedRelativePath(entry.path, "PDF artifact"), bytes: integer(entry.bytes, 8 * 1024 ** 3), sha256: entry.sha256 }; +} +function decode(bytes) { + try { + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + return invalid("PDF bundle text is not UTF-8"); + } +} +function parse(bytes) { + try { + return JSON.parse(decode(bytes)); + } catch (error) { + if (error instanceof PdfCaptureError) + throw error; + return invalid("PDF bundle JSON is invalid"); + } +} +async function inspectPdfCaptureBundle(path, options = {}) { + const maxDocument = boundedInteger(options.maxDocumentBytes, 64 * 1024 ** 2, 256 * 1024 ** 2, "maxDocumentBytes"); + const maxSource = boundedInteger(options.maxSourceBytes, 512 * 1024 ** 2, 8 * 1024 ** 3, "maxSourceBytes"); + const maxAsset = boundedInteger(options.maxAssetBytes, 100 * 1024 ** 2, 8 * 1024 ** 3, "maxAssetBytes"); + const maxTotal = boundedInteger(options.maxTotalAssetBytes, 500 * 1024 ** 2, 8 * 1024 ** 3, "maxTotalAssetBytes"); + const maxAssets = boundedInteger(options.maxVerifiedAssets, 1000, 1e4, "maxVerifiedAssets"); + const deadline = performance.now() + boundedInteger(options.maxVerificationMs, 30000, 300000, "maxVerificationMs"); + const root = await safeDirectory(path); + try { + const manifest = object(parse(await readRegularFile(root, "capture.json", 4 * 1024 ** 2, "PDF manifest", deadline))); + if (manifest.schemaVersion !== 1) + throw new PdfCaptureError("unsupported-schema", "Unsupported PDF manifest schema"); + if (manifest.kind !== "pdf" || manifest.status !== "complete" && manifest.status !== "partial") + invalid("Invalid PDF manifest kind or status"); + const document = object(manifest.document); + const pageCount = integer(document.pageCount, 1e5, 1); + const processedPages = integer(document.processedPages, pageCount, 1); + if (processedPages < pageCount && manifest.status !== "partial") + invalid("Truncated PDF cannot claim complete"); + if (manifest.quality !== undefined) { + const quality = object(manifest.quality); + if (quality.schemaVersion !== 1) + throw new PdfCaptureError("unsupported-schema", "Unsupported PDF quality schema"); + if (!Array.isArray(quality.issues) || quality.issues.length > 1e5) + invalid("Invalid PDF quality issues"); + for (const raw of quality.issues) { + const issue = object(raw); + if (!["reading-order-uncertain", "table-structure-uncertain", "math-structure-uncertain", "ocr-needed"].includes(String(issue.code))) + invalid("Unknown PDF quality issue"); + integer(issue.page, processedPages, 1); + } + if (quality.issues.length > 0 && manifest.status !== "partial") + invalid("Uncertain PDF cannot claim complete"); + } + const sourceDescriptor = descriptor(manifest.source); + if (sourceDescriptor.path !== "source.pdf" || object(manifest.source).mimeType !== "application/pdf") + invalid("Invalid retained PDF source"); + const check = async (expected, maximum) => { + if (expected.bytes > maximum) + throw new PdfCaptureError("limit-exceeded", "PDF artifact exceeds verification byte limit"); + const actual = await hashRegularFile(root, expected.path, maximum, deadline, "PDF artifact"); + return { ...actual, path: expected.path, integrity: actual.bytes === expected.bytes && actual.sha256 === expected.sha256 ? "verified" : "mismatch" }; + }; + const source = await check(sourceDescriptor, maxSource); + let expectedMarkdown = null; + let markdownPath; + if (manifest.markdown !== undefined) { + expectedMarkdown = descriptor(manifest.markdown); + markdownPath = expectedMarkdown.path; + if (markdownPath.includes("/") || !markdownPath.endsWith(".md")) + invalid("Invalid PDF Markdown path"); + } else { + const directory = await opendir(root.path); + const candidates = []; + let entries = 0; + try { + for await (const entry of directory) { + assertAssetVerificationDeadline(deadline); + if (++entries > 1000) + throw new PdfCaptureError("limit-exceeded", "Legacy PDF directory exceeds entry limit"); + if (entry.name.endsWith(".md")) + candidates.push(entry.name); + } + } finally { + try { + await directory.close(); + } catch {} + } + if (candidates.length !== 1) + invalid("Legacy PDF bundle must contain exactly one Markdown document"); + markdownPath = candidates[0]; + } + const markdownBytes = await readRegularFile(root, markdownPath, maxDocument, "PDF Markdown", deadline); + const markdownText = decode(markdownBytes); + const markdownDigest = digest(markdownBytes); + const markdown = { + path: markdownPath, + bytes: markdownBytes.length, + sha256: markdownDigest, + markdown: markdownText, + integrity: expectedMarkdown === null ? "unavailable" : expectedMarkdown.bytes === markdownBytes.length && expectedMarkdown.sha256 === markdownDigest ? "verified" : "mismatch" + }; + const assetReferences = new Map; + let currentPage = 0; + for (const match of markdownText.matchAll(/|!\[[^\]\n]*\]\((assets\/[^)\r\n]+)\)/gu)) { + if (match[1] !== undefined) + currentPage = integer(Number(match[1]), processedPages, 1); + else if (match[2] !== undefined) { + const pages = assetReferences.get(match[2]) ?? new Set; + pages.add(currentPage); + assetReferences.set(match[2], pages); + } + } + const anchors = new Set; + let anchorCount = 0; + for (const match of markdownText.matchAll(//gu)) { + if (++anchorCount > 1e5) + invalid("Too many PDF page anchors"); + anchors.add(integer(Number(match[1]), processedPages, 1)); + } + if (!Array.isArray(manifest.images) || manifest.images.length > maxAssets) + invalid("Invalid PDF image count"); + const imageIds = new Map; + const assetsByPath = new Map; + const declaredPages = new Map; + let totalBytes = 0; + for (const raw of manifest.images) { + const image = object(raw); + if (typeof image.id !== "string" || !/^[a-z0-9][a-z0-9._:-]{0,255}$/u.test(image.id) || imageIds.has(image.id)) + invalid("Invalid or duplicate PDF image ID"); + const page = integer(image.page, processedPages, 1); + if (!anchors.has(page)) + invalid("PDF image has no corresponding Markdown page anchor"); + const asset = descriptor(image.asset); + if (!/^assets\/[^/]+$/u.test(asset.path)) + invalid("PDF image must be a direct assets child"); + if (!assetReferences.get(asset.path)?.has(page)) + invalid("PDF image has no corresponding Markdown asset reference"); + imageIds.set(image.id, asset.sha256); + const pages = declaredPages.get(asset.path) ?? new Set; + pages.add(page); + declaredPages.set(asset.path, pages); + const existing = assetsByPath.get(asset.path); + if (existing !== undefined) { + if (existing.bytes !== asset.bytes || existing.sha256 !== asset.sha256) + invalid("Conflicting PDF asset descriptors"); + } else { + totalBytes += asset.bytes; + if (totalBytes > maxTotal) + throw new PdfCaptureError("limit-exceeded", "PDF assets exceed total byte limit"); + const checked = await check(asset, Math.min(maxAsset, maxTotal - (totalBytes - asset.bytes))); + totalBytes += checked.bytes - asset.bytes; + assetsByPath.set(asset.path, checked); + } + } + for (const [path2, pages] of assetReferences) { + if (!assetsByPath.has(path2)) + invalid("Markdown references an undeclared PDF asset"); + if ([...pages].some((page) => !declaredPages.get(path2)?.has(page))) + invalid("Markdown asset page disagrees with PDF manifest"); + } + const extraction = object(manifest.extraction); + if (integer(extraction.imageCount, maxAssets) !== imageIds.size) + invalid("PDF image count disagrees with manifest"); + let annotations = null; + if (manifest.annotations !== null) { + const annotationDescriptor = descriptor(manifest.annotations); + if (annotationDescriptor.path !== "annotations.json") + invalid("Invalid PDF annotations path"); + const annotationBytes = await readRegularFile(root, "annotations.json", 16 * 1024 ** 2, "PDF annotations", deadline); + const annotationHash = digest(annotationBytes); + annotations = { path: annotationDescriptor.path, bytes: annotationBytes.length, sha256: annotationHash, integrity: annotationDescriptor.bytes === annotationBytes.length && annotationDescriptor.sha256 === annotationHash ? "verified" : "mismatch" }; + let retained; + try { + retained = parsePdfImageInterpretations(parse(annotationBytes)); + } catch (error) { + if (error instanceof PdfCaptureError) + throw error; + throw new PdfCaptureError("invalid-input", "Invalid PDF image annotations", { cause: error }); + } + if (retained.length !== integer(object(manifest.annotations).count, 1e4)) + invalid("PDF annotation count disagrees with manifest"); + for (const annotation of retained) { + if (imageIds.get(annotation.id) !== annotation.sha256) + invalid("PDF annotation references an unknown image or digest"); + } + } + await assertBundleRoot(root, deadline); + return { root: root.path, schemaVersion: 1, status: manifest.status, pageCount, processedPages, markdown, source, assets: [...assetsByPath.values()], annotations }; + } finally { + await root.handle.close(); + } +} +async function verifyPdfCaptureBundle(path, options = {}) { + const inspection = await inspectPdfCaptureBundle(path, options); + const issues = []; + const add = (file, kind) => { + if (file.integrity !== "verified") + issues.push({ kind, path: file.path, integrity: file.integrity }); + }; + add(inspection.markdown, "document-integrity"); + add(inspection.source, "source-integrity"); + for (const asset of inspection.assets) + add(asset, "asset-integrity"); + if (inspection.annotations !== null) + add(inspection.annotations, "annotation-integrity"); + return { ok: issues.length === 0, inspection, issues }; +} + // src/pdf.ts var PDF_CAPTURE_MANIFEST_FILENAME2 = PDF_CAPTURE_MANIFEST_FILENAME; var PDF_CAPTURE_MANIFEST_SCHEMA_VERSION2 = PDF_CAPTURE_MANIFEST_SCHEMA_VERSION; @@ -58,7 +486,11 @@ var resolvePdfTools2 = resolvePdfTools; var runPdfCapture2 = runPdfCapture; var runPdfCommand = main; var runPdfToolCommand2 = runPdfToolCommand; +var inspectPdfCaptureBundle2 = (...arguments_) => inspectPdfCaptureBundle(...arguments_); +var verifyPdfCaptureBundle2 = (...arguments_) => verifyPdfCaptureBundle(...arguments_); +var PdfCaptureError2 = PdfCaptureError; export { + verifyPdfCaptureBundle2 as verifyPdfCaptureBundle, runPdfToolCommand2 as runPdfToolCommand, runPdfCommand, runPdfCapture2 as runPdfCapture, @@ -76,9 +508,11 @@ export { parsePdfArguments2 as parsePdfArguments, ocrPdfImage2 as ocrPdfImage, layoutBlocks2 as layoutBlocks, + inspectPdfCaptureBundle2 as inspectPdfCaptureBundle, inspectPdf2 as inspectPdf, decodePopplerText2 as decodePopplerText, buildPdfMarkdown2 as buildPdfMarkdown, + PdfCaptureError2 as PdfCaptureError, PDF_CAPTURE_SOURCE_FILENAME2 as PDF_CAPTURE_SOURCE_FILENAME, PDF_CAPTURE_MANIFEST_SCHEMA_VERSION2 as PDF_CAPTURE_MANIFEST_SCHEMA_VERSION, PDF_CAPTURE_MANIFEST_FILENAME2 as PDF_CAPTURE_MANIFEST_FILENAME, diff --git a/dist/support-hd4754m4.js b/dist/support-s059eytv.js similarity index 90% rename from dist/support-hd4754m4.js rename to dist/support-s059eytv.js index ec166e51..861fb99b 100644 --- a/dist/support-hd4754m4.js +++ b/dist/support-s059eytv.js @@ -2,24 +2,25 @@ import { parseArguments as parseArguments2, parseUrlMetadataArguments -} from "./index-878bh2f1.js"; -import"./index-5h5awwh4.js"; +} from "./index-30v9w4cy.js"; +import"./index-b40e2dkx.js"; import { parsePdfArguments -} from "./index-m9p2a4mj.js"; +} from "./index-my50xsa3.js"; import"./index-3hts6yy9.js"; -import"./index-j4zgmzjr.js"; -import"./index-npg9z1a4.js"; -import"./index-h9vyd03z.js"; -import"./index-hacpnaew.js"; -import"./index-k86wepd8.js"; -import"./index-nd8v7r0z.js"; +import"./index-8djdwjjd.js"; +import"./index-xn5q4kne.js"; +import"./index-58azscg5.js"; +import"./index-fd8j527z.js"; +import"./index-h2w7syy2.js"; +import"./index-sp43012q.js"; import"./index-hnvx7m1j.js"; import"./index-t4hjm1kk.js"; -import"./index-ncwzmsge.js"; -import"./index-cd75vky9.js"; +import"./index-nwdzhj1h.js"; +import"./index-jbtyp6q5.js"; import"./index-2gv8y733.js"; -import"./index-mxxxytys.js"; +import"./index-g63d3937.js"; +import"./index-e2wstc7t.js"; import"./index-w2zc0vwa.js"; import"./index-e5fbsywq.js"; import"./index-gh719d91.js"; diff --git a/dist/url-intelligence.js b/dist/url-intelligence.js index 546090e0..cc33313c 100644 --- a/dist/url-intelligence.js +++ b/dist/url-intelligence.js @@ -10,7 +10,7 @@ import { readUrlMetadataDocument, renderUrlMetadataDocument, writeUrlMetadataDocument -} from "./index-5h5awwh4.js"; +} from "./index-b40e2dkx.js"; import { ARCHIVE_TODAY_HOSTS, ArchiveTodayFailure, @@ -23,8 +23,9 @@ import { parseMetadataSearchResponse, rankMetadataSearchResults, selectNewestArchiveTodayMemento -} from "./index-hacpnaew.js"; -import"./index-mxxxytys.js"; +} from "./index-fd8j527z.js"; +import"./index-g63d3937.js"; +import"./index-e2wstc7t.js"; import"./index-e5fbsywq.js"; import"./index-gh719d91.js"; import"./index-1xxnjn0d.js"; diff --git a/docs/capture.md b/docs/capture.md index 2e7d71bf..bc1afe62 100644 --- a/docs/capture.md +++ b/docs/capture.md @@ -235,3 +235,22 @@ This operation discloses each eligible source URL as an exact search query to th Only a result with the same conservative URL identity can supply the selected title or description. A timestamped Archive.today result must embed that same source identity before it can be recorded. Provider failures, partial engine coverage, throttling, and absence remain explicit in the sidecar. A zero-result record is `not-found` only after complete attempted coverage; any failed engine or archive lookup keeps the top-level result `partial`. The command skips compatible sidecars by default. Use `--refresh` to query them again, `--no-archive` to omit Archive.today discovery, or `--delay-ms` to change the default one-second interval between outbound requests. It reads Markdown and sidecars through validated no-follow descriptors, carries the source Markdown and article-directory identities through provider work, and revalidates them under an inode-bound kernel advisory lock immediately before atomic installation. A live writer's lock cannot be stolen. The kernel releases a crashed writer's lock, and the next run safely reuses its verified lock file while UUID-scoped temporary names prevent an orphan from blocking progress. Malformed, linked, mismatched, replaced, or concurrently changed sources, locks, and sidecars fail closed. + +## Observed article metadata + +New captures include an optional `metadata` envelope in `capture.json`. Schema 1 +keeps each observed value, its normalized value, source and locator, the selection +rule, and whether that claim was selected. Conflicting claims remain available. +The parser reads HTML title, language, standard meta and link tags, and bounded +JSON-LD from the captured document. It does not fetch remote JSON-LD contexts. + +Specific article metadata takes precedence over generic webpage metadata. +Multiple authors and alternate links can remain selected. A date without an +explicit timezone stays local; a calendar date is not converted into an instant. +Metadata URLs omit query strings, fragments and credentials. They are descriptive +evidence and must not replace the original acquisition URL. Parser limits and +malformed JSON-LD appear in the envelope's warnings. Older bundles can omit the +envelope. + +`@hraness/wordcell/capture` exports `extractArticleMetadata` and +`parseArticleMetadata`; the bundle reader returns this evidence when present. diff --git a/docs/getting-started.md b/docs/getting-started.md index 9c157b5b..d4925a03 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -13,13 +13,13 @@ Install [Bun 1.3.14 or newer](https://bun.sh/docs/installation) and Git first, then install the pinned release archive: ```sh -bun add --global --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.26.3/hraness-wordcell-0.26.3.tgz +bun add --global --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.27.0/hraness-wordcell-0.27.0.tgz wordcell --help ``` If your shell cannot find `wordcell`, add your package manager's global executable directory to `PATH` and reopen the terminal. The npm mirror -`npm install --global --ignore-scripts @hraness/wordcell@0.26.3` carries the +`npm install --global --ignore-scripts @hraness/wordcell@0.27.0` carries the same bytes. ## Search an existing Markdown or Obsidian vault diff --git a/docs/pdf.md b/docs/pdf.md index 7b3ae29b..22718504 100644 --- a/docs/pdf.md +++ b/docs/pdf.md @@ -110,3 +110,33 @@ Reaching any boundary is recorded and makes the result `partial`. 5. Inspect low-confidence OCR against its source image. 6. Keep later summaries and interpretations in maintained notes linked to the capture. + +## Layout quality and verification + +Separated columns are read in sequence before lines are merged. Compact aligned +tables with an explicit header and numeric cells become Markdown tables. Basic +raised digits remain superscripts. Complex formulas, ambiguous tables and +uncertain layout produce page-specific quality issues and a partial capture; +the original PDF remains the reference. Repeated headers are removed only near +page margins. These rules do not promise complete layout reconstruction. + +New bundles bind the Markdown bytes as well as the original PDF and images. +Use the public SDK to inspect or verify an existing bundle: + +```ts +import { verifyPdfCaptureBundle } from "@hraness/wordcell/pdf"; +const result = await verifyPdfCaptureBundle("/absolute/path/to/saved-document"); +if (!result.ok) throw new Error("PDF bundle verification failed"); +``` + +`inspectPdfCaptureBundle` returns the saved text and integrity results. +`verifyPdfCaptureBundle` checks bounded, confined files and page/asset references. +Legacy bundles without a Markdown digest report unavailable integrity rather +than verified content. `PdfCaptureError.code` provides stable failure categories; +unknown failures remain unknown, and password errors require a recognized tool +diagnostic. + +A host with its own process supervisor can inject `runTool` and call +`runPdfToolCommand({ ...command, processGroup: "inherit" })`. That host owns +termination and descendant cleanup for the surrounding group. Standalone +commands keep the default separate process group. diff --git a/docs/reference.md b/docs/reference.md index c694fa8c..aa58d96d 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -11,7 +11,7 @@ Copy this prompt into Codex, Claude Code, or another coding agent: ```text -Install the `wordcell` Agent Skill from `hraness/wordcell#v0.26.3` with the standard skills +Install the `wordcell` Agent Skill from `hraness/wordcell#v0.27.0` with the standard skills CLI. Use the skill's runtime instructions to install the exact versioned GitHub release archive only when the command is missing. Verify it with `wordcell doctor` and `wordcell --help`, but do not initialize or modify a vault until @@ -21,8 +21,8 @@ I ask. Install the single public skill with either runner: ```sh -npx skills add hraness/wordcell#v0.26.3 -bunx skills add hraness/wordcell#v0.26.3 +npx skills add hraness/wordcell#v0.27.0 +bunx skills add hraness/wordcell#v0.27.0 ``` Both commands discover the same `wordcell` skill and install it into the selected @@ -39,7 +39,7 @@ installed skill is byte-identical to the repository source. Install the two global commands with Bun: ```sh -bun add --global --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.26.3/hraness-wordcell-0.26.3.tgz +bun add --global --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.27.0/hraness-wordcell-0.27.0.tgz wordcell --help wordcell-evaluation-builder --help ``` @@ -47,7 +47,7 @@ wordcell-evaluation-builder --help The same GitHub archive can be installed with npm: ```sh -npm install --global --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.26.3/hraness-wordcell-0.26.3.tgz +npm install --global --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.27.0/hraness-wordcell-0.27.0.tgz wordcell --help ``` @@ -60,7 +60,7 @@ reviewed and enabled; run `wordcell doctor` to inspect the resulting capabilitie For programmatic use, add the versioned GitHub archive to a Bun project: ```sh -bun add --exact --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.26.3/hraness-wordcell-0.26.3.tgz +bun add --exact --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.27.0/hraness-wordcell-0.27.0.tgz ``` The resulting dependency should remain exact: @@ -68,7 +68,7 @@ The resulting dependency should remain exact: ```json { "dependencies": { - "@hraness/wordcell": "https://github.com/hraness/wordcell/releases/download/v0.26.3/hraness-wordcell-0.26.3.tgz" + "@hraness/wordcell": "https://github.com/hraness/wordcell/releases/download/v0.27.0/hraness-wordcell-0.27.0.tgz" } } ``` @@ -730,9 +730,9 @@ companion skill for a distinct recurring ritual. The package smoke test keeps future tagged packages byte-identical to that source tree. ```sh -npx skills add hraness/wordcell#v0.26.3 +npx skills add hraness/wordcell#v0.27.0 # or -bunx skills add hraness/wordcell#v0.26.3 +bunx skills add hraness/wordcell#v0.27.0 ``` The skill invokes the installed `wordcell` command without depending on a repository diff --git a/package.json b/package.json index e35a5f62..15d181e5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hraness/wordcell", - "version": "0.26.3", + "version": "0.27.0", "description": "Wordcell keeps decisions, plans, and sources as Markdown beside your code, so coding agents can find them from the file they are about to change.", "license": "MIT", "type": "module", @@ -484,7 +484,10 @@ "src/cli-update.ts", "src/cli-release.ts", "src/evaluation-builder-bin.ts", - "src/evaluation-builder-cli-args.ts" + "src/evaluation-builder-cli-args.ts", + "src/clip/metadata.ts", + "src/pdf/bundle-reader.ts", + "src/pdf/errors.ts" ], "scripts": { "kb:refresh": "bunx --bun --package https://github.com/hraness/wordcell/releases/download/v0.24.1/hraness-wordcell-0.24.1.tgz wordcell refresh --root kb", @@ -511,16 +514,17 @@ "check:worker": "tsc -p worker/tsconfig.json --noEmit" }, "dependencies": { + "@hraness/cli-update": "https://github.com/hraness/cli-update/releases/download/v0.1.0/hraness-cli-update-0.1.0.tgz", "@hraness/oh": "https://github.com/hraness/oh/releases/download/v0.12.0/hraness-oh-0.12.0.tgz", + "@hraness/support-foundation": "git+https://github.com/hraness/support-foundation.git#b32c1c81bb2444f50509ed54388758ecfab1f1c0", "@steipete/sweet-cookie": "0.4.3", "@tobilu/qmd": "https://github.com/hraness/qmd/releases/download/v2.8.3-hraness.1/hraness-qmd-2.8.3-hraness.1.tgz", "agent-browser": "0.32.3", "defuddle": "0.19.1", - "yaml": "^2.9.0", - "zod": "^4.4.3", "effect": "3.22.1", - "@hraness/support-foundation": "git+https://github.com/hraness/support-foundation.git#b32c1c81bb2444f50509ed54388758ecfab1f1c0", - "@hraness/cli-update": "https://github.com/hraness/cli-update/releases/download/v0.1.0/hraness-cli-update-0.1.0.tgz" + "entities": "7.0.1", + "yaml": "^2.9.0", + "zod": "^4.4.3" }, "devDependencies": { "@hraness/build-governance": "github:hraness/build-governance#v0.1.0", diff --git a/portfolio-inventory.json b/portfolio-inventory.json index b3815b7c..b84b7a6b 100644 --- a/portfolio-inventory.json +++ b/portfolio-inventory.json @@ -8,7 +8,7 @@ "name": "@hraness/wordcell", "path": ".", "visibility": "public", - "version": "0.26.3" + "version": "0.27.0" } ], "dependencies": [ diff --git a/scripts/kb-skill-contract.test.ts b/scripts/kb-skill-contract.test.ts index 6881fb32..81650954 100644 --- a/scripts/kb-skill-contract.test.ts +++ b/scripts/kb-skill-contract.test.ts @@ -194,7 +194,7 @@ test("the shipped skill resources preserve routing and companion contracts", asy "references/url-platforms.md", "templates/companion-skill.template.md", ]); - expect(manifest.version).toBe("0.26.3"); + expect(manifest.version).toBe("0.27.0"); const scripts = manifest.scripts as Record; expect(scripts["kb:search"]).not.toContain("--rerank"); expect(scripts["kb:search:clef"]).toContain("--rerank clef"); diff --git a/scripts/kb-skill-contract.ts b/scripts/kb-skill-contract.ts index 455ee702..dbf04c2b 100644 --- a/scripts/kb-skill-contract.ts +++ b/scripts/kb-skill-contract.ts @@ -418,8 +418,8 @@ export function validateKbSkillContractResources( errors.push(`SKILL.md must link ${link}`); } } - if (!resources.skill.includes("bun add --global --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.26.3/hraness-wordcell-0.26.3.tgz")) { - errors.push("SKILL.md must retain the immutable 0.26.3 GitHub runtime pin"); + if (!resources.skill.includes("bun add --global --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.27.0/hraness-wordcell-0.27.0.tgz")) { + errors.push("SKILL.md must retain the immutable 0.27.0 GitHub runtime pin"); } for (const [name, contents, headings] of [ ["customize.md", resources.customize, CUSTOMIZE_HEADINGS], diff --git a/scripts/npm-release-workflow.test.ts b/scripts/npm-release-workflow.test.ts index f3c60645..d60d5480 100644 --- a/scripts/npm-release-workflow.test.ts +++ b/scripts/npm-release-workflow.test.ts @@ -382,7 +382,7 @@ describe("npm release workflows", () => { readonly version?: unknown; }; expect(manifest).toEqual(expect.objectContaining({ - version: "0.26.3", + version: "0.27.0", description: expect.any(String), keywords: [ "knowledge-base", diff --git a/scripts/package-artifact.ts b/scripts/package-artifact.ts index 8871d67f..c3a8f918 100644 --- a/scripts/package-artifact.ts +++ b/scripts/package-artifact.ts @@ -4,18 +4,18 @@ import { gunzipSync } from "node:zlib"; const blockSize = 512; const packagePrefix = "package/"; -const maximumTarBytes = 6_950_000; +const maximumTarBytes = 7_050_000; const ustarSignature = Buffer.from([0x75, 0x73, 0x74, 0x61, 0x72, 0x00, 0x30, 0x30]); -// The Clef provider migration ships src/rerank-clef.ts and dist/rerank-clef.js: -// measured npm package 6,385,735 unpacked bytes (1,439,073 compressed, 302 files). -// Preserve the compressed/file bounds and the existing 450 KB tar overhead cap. -export const maximumUnpackedBytes = 6_500_000; +// Metadata/PDF SDK0.27.0 ships three public transitive source modules: +// measured npm package6,492,562 unpacked bytes (1,463,933 compressed,306 files). +// Reviewed modest headroom preserves the450KB tar overhead cap. +export const maximumUnpackedBytes = 6_600_000; const packageBudget = Object.freeze({ entryCount: { min: 190, max: 420 }, - fileCount: { min: 190, max: 306 }, - packedBytes: { min: 950_000, max: 1_450_000 }, + fileCount: { min: 190, max: 310 }, + packedBytes: { min: 950_000, max: 1_500_000 }, unpackedBytes: { min: 4_500_000, max: maximumUnpackedBytes }, }); diff --git a/scripts/package-smoke.ts b/scripts/package-smoke.ts index 72f76f42..a4555905 100644 --- a/scripts/package-smoke.ts +++ b/scripts/package-smoke.ts @@ -12,7 +12,7 @@ import { requiresOhAdoptionPreparerExport } from "./npm-package-identity.js"; const packageName = "@hraness/wordcell"; const maximumPackageFiles = 306; -const maximumPackedBytes = 1_450_000; +const maximumPackedBytes = 1_500_000; const importSpecifiers = [ "@hraness/wordcell", "@hraness/wordcell/agent-context", @@ -77,6 +77,8 @@ const baselineRequiredNamedExports = { "@hraness/wordcell/graph-authority": ["openGraphAuthority", "queryGraph", "rebuildGraph", "verifyGraph"], "@hraness/wordcell/graph-percolation": ["percolateWithGraph"], "@hraness/wordcell/clip/bundle-reader": ["readCaptureBundle", "verifyCaptureBundle"], + "@hraness/wordcell/capture": ["extractArticleMetadata", "parseArticleMetadata"], + "@hraness/wordcell/pdf": ["inspectPdfCaptureBundle", "verifyPdfCaptureBundle", "PdfCaptureError"], "@hraness/wordcell/clip/jobs": ["createCaptureJob", "openCaptureJobStore", "updateCaptureJob"], "@hraness/wordcell/clip/refresh": ["diffCaptureBundle"], "@hraness/wordcell/portfolio": ["openKnowledgePortfolio", "parsePortfolioRegistry", "parseQualifiedDocumentUri"], diff --git a/site/app/docs/docs.generated.ts b/site/app/docs/docs.generated.ts index 340deb79..c2f7ca30 100644 --- a/site/app/docs/docs.generated.ts +++ b/site/app/docs/docs.generated.ts @@ -5,8 +5,8 @@ export const docHtml: Readonly> = { "agent-handoffs": "

Connect Wordcell to your coding agent

\n

wordcell mcp serves one Markdown vault to a coding agent over the Model\nContext Protocol (MCP). The agent starts the server itself, then searches,\nreads, and follows links between your notes, and it can write notes unless you\nstart the server with --read-only. This guide registers the server with\nClaude Code, Codex, and Cursor, checks that the agent can start it, and lists\nthe errors you are most likely to see.

\n

Checked on October 4, 2026: Claude Code 2.1.287 and Codex CLI 0.160.0\nregistered Wordcell 0.26.3 with the commands below, in a test project with\nempty configuration directories. The command output on this page comes from\nthose runs and from calling the server's tools directly, without an agent.

\n

Before you start

\n
    \n
  1. Install Wordcell 0.23.0 or newer, the first release with wordcell mcp.\nGet started has the install command.
  2. \n
  3. Find the absolute path of your vault, such as a kb/ folder in your\nrepository or an existing notes folder.
  4. \n
  5. Search the vault from a terminal for a phrase you know is in one note:
  6. \n
\n
wordcell search "a phrase from your notes" --root /absolute/path/to/kb --mode exact\n
\n

If the result names the note you expected, the vault and the installation\nwork, and any later problem is in the agent's configuration. Exact search\nreads the Markdown directly and needs no index or model.

\n

Every client runs the same server command. Choose its arguments first:

\n\n

Use absolute paths. A client can start the server from a different working\ndirectory than your terminal's, and a relative path resolves from there.

\n

Claude Code

\n

Add the server with claude mcp add. Everything after -- is the server\ncommand:

\n
claude mcp add --transport stdio --scope project wordcell -- wordcell mcp --root /absolute/path/to/kb\n
\n

--scope decides where Claude Code saves the server, as its\nMCP guide describes:

\n\n\n\n\n\n\n\n\n\n
ScopeSaved inAvailable to
local, the default~/.claude.json, under this projectYou, in this project
project.mcp.json at the project rootEveryone who uses the repository, after each person approves it
user~/.claude.jsonYou, in every project
\n

claude mcp list starts each configured server to check it. A server that\nstarted prints ✔ Connected:

\n
$ claude mcp list\nChecking MCP server health…\n\nwordcell: wordcell mcp --root /tmp/wordcell-demo/kb --repo /tmp/wordcell-demo - ✔ Connected\n
\n

A server saved with --scope project waits until you start claude in that\nproject and approve it. Until then the list shows:

\n
wordcell: wordcell mcp --root /tmp/wordcell-demo/kb - ⏸ Pending approval (run `claude` to approve)\n
\n

An absolute vault path in .mcp.json works only on your machine. When the\nvault is a kb/ folder in the repository, a shared .mcp.json can name it\nthrough the CLAUDE_PROJECT_DIR variable that Claude Code sets for the\nserver. Keep the :-. default, which Claude Code requires for this variable\nin .mcp.json:

\n
{\n  "mcpServers": {\n    "wordcell": {\n      "type": "stdio",\n      "command": "wordcell",\n      "args": ["mcp", "--root", "${CLAUDE_PROJECT_DIR:-.}/kb"]\n    }\n  }\n}\n
\n

With the server approved, claude mcp list showed this entry connected in the\nsame test.

\n

Codex

\n
codex mcp add wordcell -- wordcell mcp --root /absolute/path/to/kb\n
\n

The command adds this table to ~/.codex/config.toml. The\nCodex MCP guide says the Codex CLI,\nthe IDE extension, and the ChatGPT desktop app share that file. You can also\nwrite the table yourself, there or in .codex/config.toml inside a trusted\nproject:

\n
[mcp_servers.wordcell]\ncommand = "wordcell"\nargs = ["mcp", "--root", "/absolute/path/to/kb"]\n
\n

codex mcp list shows the configuration without starting the server:

\n
$ codex mcp list\nName      Command   Args                                                        Env  Cwd  Status   Auth\nwordcell  wordcell  mcp --root /tmp/wordcell-demo/kb --repo /tmp/wordcell-demo  -    -    enabled  Unsupported\n
\n

enabled means the entry is switched on, not that the server works: a vault\npath that does not exist also shows enabled. Start codex and run /mcp to\nsee the servers that started. By default Codex waits 10 seconds for a server\nto start and 60 seconds for a tool call; startup_timeout_sec and\ntool_timeout_sec in the same table change those limits.

\n

Cursor

\n

Cursor reads MCP servers from .cursor/mcp.json in a project, or from\n~/.cursor/mcp.json for every project, as its\nMCP guide describes:

\n
{\n  "mcpServers": {\n    "wordcell": {\n      "type": "stdio",\n      "command": "wordcell",\n      "args": ["mcp", "--root", "/absolute/path/to/kb"]\n    }\n  }\n}\n
\n

When the vault is a kb/ folder in the project, the project file can use\nCursor's ${workspaceFolder} variable instead of a fixed path:\n"args": ["mcp", "--root", "${workspaceFolder}/kb"]. Turn servers on or off\nfrom Customize in the sidebar. Server errors appear in the Output panel under\nMCP Logs, and Cursor asks before it runs an MCP tool unless your run mode\nallows it.

\n

These steps come from Cursor's documentation, read on October 4, 2026. They\nwere not run in Cursor for this guide.

\n

Claude Desktop

\n

From the Claude menu, open Settings, then Developer, then Edit Config, and add\nthe server to claude_desktop_config.json. The file is in\n~/Library/Application Support/Claude/ on macOS and %APPDATA%\\Claude\\ on\nWindows. Quit and restart Claude Desktop after you save it.

\n
{\n  "mcpServers": {\n    "wordcell": {\n      "command": "wordcell",\n      "args": ["mcp", "--root", "/absolute/path/to/kb"]\n    }\n  }\n}\n
\n

On macOS, the server's standard error goes to\n~/Library/Logs/Claude/mcp-server-wordcell.log. These steps follow the\nlocal server guide,\nread on October 4, 2026, and were not run for this guide.

\n

Other MCP clients

\n

A client that starts local stdio servers can run Wordcell with the command\nwordcell and the arguments mcp --root /absolute/path/to/kb. The server\nwrites only protocol messages to standard output and its errors to standard\nerror. The local MCP server reference lists\nthe protocol versions it accepts.

\n

Check that the agent reads your notes

\n

Ask the agent to call Wordcell's search tool with the phrase you searched in\nthe terminal, in exact mode:

\n
{"query": "a phrase from your notes", "mode": "exact", "limit": 3}\n
\n

The result should name the same note as the terminal search. Then ask for\nget_note with that note's ID, its vault path without .md, and compare the\nbody with the file. Called directly in the test project, search for\nparser retries returned notes/parser-contract at line 11, and context for\nsrc/parser.ts returned the same note, which declares that file in its\nrepository_scopes.

\n

Start with exact mode. The default mode, hybrid, adds local semantic search,\nand the first hybrid search on a machine downloads an embedding model. In one\nrun on October 4, 2026, that download was 334 MB and the first call took 26.5\nseconds. When semantic search is unavailable, the result is marked partial and\ndiagnostics.lanes gives the reason.

\n

The tools the agent sees depend on the server arguments:

\n\n\n\n\n\n\n\n\n\n
Server argumentsTools
--rootsearch, list_notes, get_note, backlinks, links, create_note, update_note_body, add_relation
--root and --repoThe same tools and context
--root, --repo, and --read-onlysearch, context, list_notes, get_note, backlinks, links
\n

Keep results within the agent's limits

\n

Wordcell caps each tool result at 64 KiB of JSON and sends the same JSON again\nas text. When the cap applies, it leaves out the items that do not fit and\nsets truncated: true and an omitted count. On a test vault of 400 notes:

\n\n\n\n\n\n\n\n\n\n\n
CallJSON returned as text
search, exact, default 10 results13.2 KiB
search, exact, limit: 34.3 KiB
search, exact, limit: 10062.9 KiB, with 49 results returned and 51 omitted
list_notes, limit: 100063.7 KiB, with 142 of 400 notes returned
\n

Claude Code warns when an MCP tool result passes 10,000 tokens and limits\nresults to 25,000 tokens by default. Set MAX_MCP_OUTPUT_TOKENS before you\nstart claude to raise the limit; a result over the limit is saved to a file\nthat Claude reads when it needs to, according to its\noutput limits.\nClaude Code also saves any text result longer than 50,000 characters to a\nfile, whatever MAX_MCP_OUTPUT_TOKENS says, so the two largest results in the\ntable reach Claude as a file path.\nCodex takes a per-tool output_token_limit in a table such as\n[mcp_servers.wordcell.tools.search] in config.toml, as its\nMCP guide describes. Token counts\ndepend on each model's tokenizer and were not measured here, so keep limit\nat 10 or lower and open notes one at a time with get_note.

\n

Troubleshoot the connection

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
SymptomWhat to check or change
claude mcp list shows Failed to connect with ENOENT: Executable not found in $PATH: "wordcell"The client cannot find wordcell on its PATH, which can be shorter than your terminal's. Use the absolute path that which wordcell prints as the command, then restart the client.
claude mcp list shows Failed to connect with CONNECTION_CLOSED: Connection closedThe server started and exited. Run the same command in a terminal to see why. A vault path that does not exist prints ✗ ENOENT: no such file or directory and → wordcell mcp --help, then exits with status 2. With an absolute wordcell path, Bun must also be on the client's PATH, because wordcell runs on Bun.
claude mcp list shows ⏸ Pending approval (run `claude` to approve)A server from .mcp.json waits for approval. Start claude in the project and accept it.
Codex lists the server, but its tools never appearcodex mcp list does not start servers. Run the server command in a terminal, fix the path or PATH, and check /mcp again.
Search returns a partial resultRead diagnostics.lanes, then search again with mode: "exact", which reads the current Markdown without the semantic index. Read optional adapters before you set up semantic search.
A result has truncated: trueLower limit on search or list_notes, or limit and depth on backlinks and links. Read the omitted count, then open notes one at a time with get_note.
get_note reports truncated: trueThe read stopped at 64 KiB. Open the Markdown file itself, and do not send the shortened body to update_note_body, which replaces the whole body.
context or the write tools are missingcontext needs --repo, and --read-only removes the write tools. Change the server arguments and restart the connection.
\n

Remove the server

\n\n

Next, use Wordcell with a coding agent to keep decisions\nand plans beside the code, or\nsearch an existing Obsidian vault from the agent without\nchanging it.

\n", "obsidian": "

Use Wordcell with an existing Obsidian vault

\n

Point Wordcell at the folder Obsidian already uses as a vault. Wordcell reads\nthe Markdown files where they are: it needs no index.md, no import, and no\nchange to your notes, and it does not read the .obsidian/ settings folder.\nYour coding agent can then search the vault, follow its wikilinks, and read\nnotes, with the write tools turned off if you prefer.

\n

The behavior on this page was checked on October 4, 2026 with Wordcell 0.26.3\non a test vault that has an .obsidian/ folder and one note for each link\nform in the table below. The vault was a Git repository, and git status\nshowed no changes after any command on this page, including the MCP tool\ncalls.

\n

Search the vault

\n

Wordcell 0.22.0 and newer search a folder without initializing it. Run an\nexact search with the vault's absolute path:

\n
wordcell search "retry budget" --root /absolute/path/to/vault --mode exact\n
\n

Search an existing Markdown or Obsidian vault\nlists the files a search skips and what to try when nothing matches. Exact\nsearch finds text in callouts, inline #tags, and frontmatter, and it matches\na note's aliases as well as its title. Wordcell reads only .md files, so\ncanvases and attachments are not searched.

\n

What Wordcell reads from Obsidian notes

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Obsidian syntaxExampleWhat Wordcell does
Wikilink to a note[[Parser rewrite]]Resolves the link and counts it as a backlink.
Display text[[Parser rewrite|the rewrite]]Resolves the link.
Folder path[[Projects/Parser rewrite]]Resolves the link.
Heading or block link[[Parser rewrite#Retry budget]], [[Parser rewrite#^retry-budget]]Resolves the link to the note. It does not check that the heading or block exists.
Embedded note or section![[Parser rewrite]], ![[Parser rewrite#Retry budget]]Resolves the link and counts it as a backlink.
Two notes with the same name[[Standup]]Reports an ambiguous link and names both paths. Link with the folder path instead.
Different letter case[[parser rewrite]]Reports a broken link. Match the case of the file name.
Alias as the link target[[Parser v2]]Reports a broken link and lists an unlinked mention of the note that has the alias.
Markdown link to a note[the rewrite](../Projects/Parser%20rewrite.md)Ignores it: the link is neither counted nor reported.
Embedded or linked attachment![[Diagram.png]], ![[Diagram.png|300]], ![[Spec.pdf#page=2]], [[Spec.pdf]]Finds the file by name anywhere in the vault. A missing file is an error.
Frontmatter propertiesstatus: activeFilters with wordcell list --where status=active.
Frontmatter tagstags: [project, standup/weekly]Filters with wordcell list --tag project. A nested tag matches only in full: --tag standup/weekly finds it and --tag standup does not.
Inline tags#decision, #standup/notesSearches them as text. --tag does not see them.
Callouts> [!warning] Rollback planSearches the callout's text.
Settings folder.obsidian/app.jsonDoes not read it.
\n

When you pick a note through one of its aliases, Obsidian writes the link as\n[[Parser rewrite|Parser v2]], as its\naliases guide describes, and Wordcell\nresolves that form. Only a link that names the alias alone breaks. Wordcell's\nlink graph reads wikilinks and typed relations, so a vault that links notes\nwith Markdown links has no backlinks in Wordcell. In Obsidian's own search,\ntag:inbox also matches nested tags such as #inbox/to-read\n(tags guide); Wordcell's --tag matches the\nfull tag and reads only the tags property.

\n\n

wordcell check reports broken and ambiguous wikilinks and missing\nattachments, and it changes no files. It runs on a vault without an\nindex.md:

\n
wordcell check --root /absolute/path/to/vault\n
\n

On the test vault it reported these errors:

\n
error: Links/alias-as-target.md:5: broken wikilink [[Parser v2]]\nerror: Links/ambiguous-name.md:5: ambiguous wikilink [[Standup]] (Archive/Standup, Meetings/2026/Standup)\nerror: Links/different-case.md:5: broken wikilink [[parser rewrite]]\nerror: Links/missing-note.md:5: broken wikilink [[Tokenizer benchmark]]\nerror: Areas/Inbox.md:3: missing attachment Missing diagram.png: Obsidian attachment basename was not found in the vault.\n
\n

The command exits with status 3 when it finds an error. It also lists notes\nthat nothing links to and titles or aliases mentioned without a link. Those\nare advisories: a vault with advisories and no errors exits with status 0.

\n

Obsidian's own help vault is a larger test. On a local copy of the\nobsidian-help repository at its\nSeptember 29, 2026 commit, Wordcell read the 176 notes in the English folder,\nresolved 959 links, and reported 68 broken and five ambiguous ones. Of the 68, 46\nnamed an existing note with different letter case. It found 251 of the 253\nattachments the notes reference.

\n

Serve the vault to your agent without changes

\n

Start the server with --read-only, so the agent can search and read but not\nwrite. For Claude Code:

\n
claude mcp add --transport stdio wordcell -- wordcell mcp --root /absolute/path/to/vault --read-only\n
\n

Connect Wordcell to your coding agent has the same step\nfor Codex, Cursor, and Claude Desktop, and the checks to run after it. With\n--read-only, the server offers search, list_notes, get_note,\nbacklinks, and links, and a call to create_note fails with\nUnknown tool: create_note.

\n

Note IDs are vault paths without .md, spaces included, such as\nProjects/Parser rewrite. get_note returns the note's frontmatter as JSON,\nincluding aliases, tags, and your other properties, and backlinks\nreturns the notes that link to it.

\n

Ask for mode: "exact" until you decide to use semantic search. The default\nmode, hybrid, downloads an embedding model on first use. In this check,\nWordcell kept that model and its search index under ~/.cache, outside the\nvault.

\n

Let the agent write notes

\n

Without --read-only, the agent can create notes, replace a note body at the\nrevision it read, and add typed relations. A new note is an ordinary Markdown\nfile with YAML frontmatter, and Obsidian\npicks up changes made by other programs,\nso it appears in the vault like any other note. create_note does not create\nfolders. update_note_body keeps the note's frontmatter and fails if the note\nchanged after the agent read it. Commit the vault before you turn on the write\ntools, so git diff shows each change the agent makes.

\n", "migration-from-supermemory": "

Migrate from Supermemory

\n

This guide moves the documents and memory entries you saved in Supermemory into\na Wordcell vault: Markdown notes in a Git repository that your coding agent\nreads and writes through wordcell mcp. You export your data with the\nSupermemory API, import it with wordcell import supermemory, check the\nresult, and connect your agent. Every external page linked here was checked on\nSeptember 26, 2026.

\n

wordcell import supermemory and wordcell mcp need Wordcell 0.23.0 or\nlater. Install it as Get started shows;\nthe installation reference lists the\nrequirements.

\n

Who should switch and who should stay

\n

Switch when your agents keep memory about your own work and you want that\nmemory in files. The\nSupermemory plugin for Claude Code\nsaves conversations and tool use to your Supermemory account, and\nSMFS mounts a container as a\ndirectory whose writes push to Supermemory. A vault keeps the same kind of\nmemory as Markdown files that you and your agent read, edit, diff, and review\nin Git, with no account or API key.

\n

Stay with Supermemory when you build a product that stores memory for many\nusers and you need what the hosted service runs for you:\ncontainer tags that\nisolate each user’s memories, API keys limited to chosen tags,\nanalytics endpoints,\nconnectors that sync on\ntheir own, automatic extraction of facts and\nuser profiles, and SOC 2\nType II compliance with a HIPAA business associate agreement on some plans\n(security). Wordcell has none\nof these.

\n

Supermemory local runs the\nsame engine on your own machine, so keeping data off a hosted service is not by\nitself a reason to switch. Switch when you want the Markdown files themselves\nto be the memory.

\n

Map Supermemory concepts to Wordcell

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
SupermemoryWordcellNotes
Container tagA vault per project or person, or the imported container_tag and container_tags fieldsOnly imported documents keep their tags: container_tag holds the first tag, and container_tags lists every tag when a document has several. To find every document with a tag, run wordcell list --where container_tag=<tag>, then again with --where container_tags=<tag>. Memory entries keep no tag, so when the separation matters, import each tag into its own vault or under its own --prefix.
DocumentA note under articles/ or notes/imported/See Where notes go.
Memory entry and its versionsOne note per version under notes/imported/memories/Each version has a supersedes relation to the one before it.
Updates, Extends, and Derives relations (graph memory)Relations you add with wordcell relation addSupermemory infers these links, so the importer does not convert them.
User profileA type: profile note that your agent maintainsSee Connect your agent.
Automatic extractionNotes that you or your agent writewordcell percolate suggests connections for you to review. Nothing writes a note on its own.
Forgetting and forgetAfterDelete the note (Git keeps its history) or mark it supersededThe importer skips forgotten entries and records forget_after, but no note expires.
ConnectorsExports that you save into the vaultEach source needs its own export. See Replace connectors.
Hosted MCP serverwordcell mcpsearch_memory becomes search; get_document and get_profile become get_note; add_memory becomes create_note or update_note_body; list_documents and list_memories become list_notes. list_spaces and who_am_i have no equivalent.
SMFSThe vault directorySync it with Git between machines.
REST API and SDKsThe wordcell CLI and TypeScript SDKSee the reference.
\n

Export your data

\n

You need a Supermemory API key in SUPERMEMORY_API_KEY, and either curl and\njq or Python 3. Work in a directory outside the vault, so that the export files\nare never committed.

\n

Documents come from\nlist documents\n(POST /v3/documents/list). Memory entries, with their earlier versions, come\nfrom\nlist memory entries\n(POST /v4/memories/list), which lists entries by container tag. The scripts\nbelow request one tag at a time. They find tags in the containerTags field of\nyour documents, which the list documents reference marks as deprecated, so a\nresponse may leave it out. Pass every container tag you used as an argument.\nThe scripts print the tags they export, and they stop with an error when they\nfind none.

\n

Save this script as export-supermemory.sh:

\n
#!/bin/sh\n# Save Supermemory documents and memory entries as JSON pages in the\n# current directory. Extra arguments are container tags to export\n# in addition to the tags found on your documents.\nset -eu\nbase=https://api.supermemory.ai\n\npost() {\n  curl -fsS "$base$1" \\\n    -H "Authorization: Bearer $SUPERMEMORY_API_KEY" \\\n    -H "Content-Type: application/json" \\\n    -d "$2" -o "$3"\n}\n\nlast_page() {\n  jq '.pagination.totalPages // 0' "$1"\n}\n\npage=1\nwhile :; do\n  file=$(printf 'documents-%03d.json' "$page")\n  post /v3/documents/list "{\\"includeContent\\": true, \\"limit\\": 100, \\"page\\": $page, \\"sort\\": \\"createdAt\\", \\"order\\": \\"asc\\"}" "$file"\n  [ "$page" -ge "$(last_page "$file")" ] && break\n  page=$((page + 1))\ndone\n\n{\n  jq -rn '[inputs.memories[].containerTags[]?] | unique[]' documents-*.json\n  printf '%s\\n' "$@"\n} | sort -u | sed '/^$/d' > container-tags.txt\n\nif [ ! -s container-tags.txt ]; then\n  echo "export-supermemory: no container tags found; pass your tags as arguments" >&2\n  exit 1\nfi\necho "export-supermemory: exporting memory entries for these container tags:" >&2\ncat container-tags.txt >&2\n\nn=0\nwhile read -r tag; do\n  n=$((n + 1))\n  page=1\n  while :; do\n    file=$(printf 'memories-%03d-%03d.json' "$n" "$page")\n    body=$(jq -cn --arg tag "$tag" --argjson page "$page" \\\n      '{containerTags: [$tag], limit: 100, page: $page, sort: "createdAt", order: "asc"}')\n    post /v4/memories/list "$body" "$file"\n    [ "$page" -ge "$(last_page "$file")" ] && break\n    page=$((page + 1))\n  done\ndone < container-tags.txt\n
\n

Run it with your container tags as arguments:

\n
sh export-supermemory.sh <tag>...\n
\n

Compare the tags it prints with the tags your application sends, or with the\nspaces that the list_spaces tool of the\nhosted MCP server shows. Run\nit again with any tag it missed.

\n

The script writes one file per page: documents-001.json and onward, then\nmemories-001-001.json and onward for each tag in container-tags.txt. It\nrequests 100 items per page in ascending creation order, so the pages stay\nstable while it runs, and it stops after the last page that\npagination.totalPages reports. It stops at the first failed request and\nprints the HTTP status.

\n

"includeContent": true adds each document’s full text, which the API\nreference notes can make responses much larger. Without it, each note holds\nonly the document’s summary.

\n

The API reference marks the containerTags array as required, while the\ncontainer tags page says\nthe /v4 API accepts only containerTag. If /v4/memories/list returns status\n400, replace containerTags: [$tag] with containerTag: $tag in the script\nand run it again.

\n

With Python 3 in place of curl and jq, save this script as\nexport_supermemory.py. It saves every page in one file,\nsupermemory-export.json, and takes the same tag arguments:

\n
"""Save every Supermemory document and memory entry to supermemory-export.json.\n\nExtra arguments are container tags to export in addition to the tags\nfound on your documents.\n"""\nimport json\nimport os\nimport sys\nimport urllib.request\n\nBASE = "https://api.supermemory.ai"\nKEY = os.environ["SUPERMEMORY_API_KEY"]\n\n\ndef post(path, body):\n    request = urllib.request.Request(\n        BASE + path,\n        data=json.dumps(body).encode(),\n        headers={\n            "Authorization": f"Bearer {KEY}",\n            "Content-Type": "application/json",\n            "User-Agent": "supermemory-export",\n        },\n    )\n    with urllib.request.urlopen(request) as response:\n        return json.load(response)\n\n\ndef pages(path, body):\n    page = 1\n    while True:\n        query = {**body, "limit": 100, "page": page, "sort": "createdAt", "order": "asc"}\n        result = post(path, query)\n        yield result\n        if page >= ((result.get("pagination") or {}).get("totalPages") or 0):\n            return\n        page += 1\n\n\nexport = list(pages("/v3/documents/list", {"includeContent": True}))\nfound = {tag for p in export for d in p["memories"] for tag in d.get("containerTags") or []}\ntags = sorted(found | set(sys.argv[1:]))\nif not tags:\n    sys.exit("export_supermemory: no container tags found; pass your tags as arguments")\nprint("Exporting memory entries for these container tags:", *tags, sep="\\n", file=sys.stderr)\nfor tag in tags:\n    export.extend(pages("/v4/memories/list", {"containerTags": [tag]}))\n\nwith open("supermemory-export.json", "w", encoding="utf-8") as out:\n    json.dump(export, out)\nprint(f"Saved {len(export)} pages to supermemory-export.json")\n
\n
python3 export_supermemory.py <tag>...\n
\n

Import into a vault

\n
    \n
  1. Create a vault if you do not have one: wordcell init kb.
  2. \n
  3. Preview the import:\nwordcell import supermemory documents-*.json memories-*.json --root kb --dry-run.\nWith the Python script, name supermemory-export.json in place of the two\npatterns.
  4. \n
  5. Read the summary. It counts what the import would create, update, skip,\nreport as a conflict, or reject. Each diagnostic line names a value the\nimporter dropped or a relation it left out.
  6. \n
  7. Import: run the same command without --dry-run.
  8. \n
  9. Rebuild the catalog: wordcell refresh --root kb.
  10. \n
  11. Check the vault: wordcell check --root kb.
  12. \n
\n

For example, a dry run over pages built from the importer’s\ntest fixtures\nprinted:

\n
Dry run of 11 items from 7 files: created 10, updated 0, skipped 1, conflicts 0, rejected 0. Nothing was written.\nWould add 2 supersedes relations.\ndiagnostic: memoryRelations are the service's inferred links and were not imported\ndiagnostic: notes/imported/nested-metadata: metadata.author is a nested value; dropped\ndiagnostic: notes/imported/nested-metadata: metadata key "bad key" is not a safe key; dropped\ndiagnostic: notes/imported/nested-metadata: metadata.labels is a nested value; dropped\ndiagnostic: notes/imported/zr8tqw3nhc6ypd1fgk5lmv: status is "queued"; the imported text may be incomplete\ndiagnostic: notes/imported/memories/prefers-dark-mode-in-every-editor-and-terminal: metadata.context is a nested value; dropped\ndiagnostic: notes/imported/memories/works-from-berlin-utc-1-in-winter: parent memory mem_timezone_v1 was not imported; supersedes relation omitted\n
\n

The skipped item was a forgotten memory entry. Apart from the supersedes\nrelations between memory versions, imported notes have no contextual links to\neach other, so wordcell check lists most of them as contextual orphans. That\nlist is advisory and does not fail the check.

\n

You can export and import again at any time. The importer updates notes you\nhave not edited, skips unchanged ones, and leaves a note you edited after the\nlast import unchanged, with a line such as\nconflict: <note> from <file>#<pointer> (<id>): the note was edited after the last import.\nConflicts do not change the exit status. Import again\ndescribes each outcome, and Frontmatter lists the\nfields each note records.

\n

Verify the result

\n

List the imported notes:

\n
wordcell list --root kb --where imported_from=supermemory\n
\n

Search for a phrase you remember saving:

\n
wordcell search "dark mode" --root kb --mode exact\n
\n

Follow a memory entry’s earlier versions:

\n
wordcell relation list notes/imported/memories/<slug> --root kb\n
\n

Commit the result. Skip git init when the vault is inside a project\nrepository.

\n
cd kb\ngit init -b main\ngit add -A\ngit commit -m "Import Supermemory export"\n
\n

To keep the vault current on several machines, see\nSync a vault with Git.

\n

Connect your agent

\n

wordcell mcp serves the vault to a local Model Context Protocol client over\nstandard input and output. It runs on your machine and needs no account. With\nClaude Code, run this in your project:

\n
claude mcp add --transport stdio --scope project wordcell -- wordcell mcp --root /absolute/path/to/kb\n
\n

Connect a client has the settings for Claude\nDesktop, Cursor, and Codex, and Tools lists what each\ntool accepts. Add --read-only to remove the three write tools.

\n

Supermemory builds a profile of each user on its own. In a vault, keep one\nmaintained note in its place: notes/profile.md with type: profile and two\nsections, “Stable” for lasting facts and “Recent” for current work. Ask your\nagent to read it at the start of each session with get_note and to revise it\nwith update_note_body, which takes the revision that get_note returned. The\nWordcell skill describes this convention in\nKeep a profile note.

\n

Replace connectors

\n

Supermemory connectors pull\ncontent from sources such as Google Drive, Gmail, Notion, OneDrive, GitHub, and\nwebsites, and keep it current with webhooks and scheduled syncs. Wordcell has no\nconnectors. Save what you want your agent to know as Markdown files in the\nvault, check it, and commit. Wordcell reads Markdown files without frontmatter\nas notes, and Sync a vault with Git commits and pushes on a\nschedule.

\n

Google Drive, OneDrive, or Dropbox

\n

In Google Docs, choose File, then Download, then Markdown (.md) to save one\ndocument as a Markdown file\n(Google Docs Help). Copy\nMarkdown files out of a synced OneDrive or Dropbox folder. Move the files into a\nfolder in the vault and check it:

\n
mkdir -p kb/notes/drive\nmv ~/Downloads/meeting-notes.md kb/notes/drive/\nwordcell refresh --root kb\nwordcell check --root kb\n
\n

Convert a PDF into the vault with\nwordcell pdf <file> --output kb/articles\n(Capture PDF documents).

\n

Gmail

\n

Google Takeout downloads your mail from Gmail\n(How to download your Google data),\nbut Wordcell has no mail importer. Save the messages worth keeping as notes\ninstead: ask your agent to write a note from a message with the create_note\ntool, or pipe the text into wordcell note create with --body-file -. On\nmacOS, pipe the text you copied:

\n
mkdir -p kb/notes/mail\npbpaste | wordcell note create notes/mail/launch-date --title "Launch date" --body-file - --root kb\n
\n

The folder must exist before note create writes into it.

\n

GitHub

\n

Keep the vault in the repository, as kb/ beside the code, and your notes\ntravel with every clone (Choose where the vault lives).\nwordcell context then lists the guides and notes that apply to a path:

\n
wordcell context src/index.ts --root kb --repo .\n
\n

To keep an issue from another repository, save its text as a note with the\nGitHub CLI:

\n
mkdir -p kb/notes/github\ngh issue view 42 --repo owner/repo --json body --jq .body |\n  wordcell note create notes/github/owner-repo-42 --title "owner/repo#42" --body-file - --root kb\n
\n

Web pages

\n

Supermemory’s web crawler\ncrawls a site and recrawls it on a schedule. wordcell clip captures one page\nas a Markdown bundle under articles/, and Git records each change. Save this\nscript as ~/bin/clip-pages.sh. It captures each page in a list again and\nprints what changed since the last commit:

\n
#!/bin/sh\n# Capture each page in a list again and show what changed since the\n# last commit. Each line of the list is a slug, a space, and a URL.\nvault="${1:?usage: clip-pages.sh <vault directory> <page list>}"\nlist="${2:?usage: clip-pages.sh <vault directory> <page list>}"\nPATH="$HOME/.bun/bin:$PATH"\nstatus=0\n\nwhile read -r slug url; do\n  [ -n "$url" ] || continue\n  if wordcell clip "$url" "$slug" --output "$vault/articles" --force </dev/null; then\n    wordcell capture diff "$vault/articles/$slug" --repo "$vault" </dev/null\n  else\n    echo "clip-pages: could not capture $url" >&2\n    status=1\n  fi\ndone <"$list"\n\nwordcell refresh --root "$vault" >/dev/null || status=1\nexit "$status"\n
\n

List the pages in ~/kb-pages.txt:

\n
example-domain https://example.com/\n
\n

Then run it once a day before the sync script. With cron, add this line\n(crontab -e):

\n
0 6 * * * /bin/sh "$HOME/bin/clip-pages.sh" "$HOME/kb" "$HOME/kb-pages.txt" >>"$HOME/clip-pages.log" 2>&1\n
\n

The script expects the vault to be its own repository, as in\nSync a vault with Git. Cron starts jobs with a short PATH, so the\nscript adds ~/.bun/bin. If command -v wordcell or command -v bun prints\nanother directory, add that one too. --force replaces the whole bundle, so\nedits inside a clipped file are lost on the next run. Write your own notes\nabout a page in a separate note that links to it. Wordcell captures only the\npages you list and does not follow links. Capture web content\nand wordcell clip --help cover the slug argument, --force, signed-in pages,\nand other options.

\n

Notion

\n

Notion exports a page, and optionally its subpages, as a zip file of Markdown\nand CSV files\n(Export your content).\nUnzip it into the vault:

\n
mkdir -p kb/notes/notion\nunzip ~/Downloads/notion-export.zip -d kb/notes/notion\nwordcell refresh --root kb\nwordcell check --root kb\n
\n

Search finds the pages right away. Wordcell ignores the CSV files, and\nwordcell check may list the pages as orphans, which is advice, not an error.

\n

What does not transfer

\n

The hosted features named in\nWho should switch and who should stay\nhave no Wordcell equivalent. Neither do these:

\n\n", - "capture": "

Capture web content

\n

wordcell clip saves public and signed-in web content as an auditable Markdown bundle. It combines bounded structured adapters, HTTP extraction, browser rendering, a read-only Archive.today fallback, localized assets, and explicit completeness metadata.

\n

Check local capabilities

\n

Run the diagnostics before using local search, browser state, PDF ingestion, or video capture:

\n
wordcell doctor\nwordcell adapters\n
\n

wordcell doctor --json reports the installed runtime, QMD and static semantic-search\nprerequisites, extraction dependencies, browser support, profile display names,\nyt-dlp, ffmpeg, Poppler (pdfinfo and pdftohtml), and Tesseract. The semantic\nreport distinguishes model-free keyword readiness from SQLite and sqlite-vec\nvector prerequisites; it does not inspect or download the embedding model. The\nPoppler pair is required by wordcell pdf; Tesseract adds local OCR for scans and\nscreenshots.\nwordcell adapters --json returns the current platform capability matrix.

\n

Capture or inspect a page

\n
wordcell clip https://example.com/article\nwordcell inspect https://example.com/article\nwordcell inspect https://example.com/article --json\n
\n

The default route tries stable public structured data when available, bounded HTTP extraction, and a rendered browser when the platform or result requires one. If those routes produce no usable representation, Wordcell may make one read-only lookup for the exact URL through Archive.today's fixed archive.ph/newest/ route. An authentication, paywall, CAPTCHA, access-control, or rate-limit response disables that fallback instead of using an archive to bypass the current source's controls. Wordcell never submits the URL for archiving, retries through aliases, or lets archived HTML displace a complete or partial Hacker News or Bluesky structured capture. It validates the source through the public-network boundary before disclosure, shares one deadline across validation and provider requests, and binds every snapshot and redirect to the exact source. An archived result is rescored and always reported as partial; it keeps the original canonical URL and records the timestamped snapshot as its acquisition URL. Inspection returns the selected Markdown and capture report without writing artifacts.

\n

By default, a capture writes kb/articles/<slug>/:

\n
<slug>/\n  <slug>.md\n  capture.json\n  url-metadata.json # after an optional metadata backfill\n  assets/\n  evidence/       # only when requested\n
\n

The Markdown records source and capture metadata. capture.json records the acquisition attempts, selected extractor, scope, status, item counts, warnings, asset hashes, and requested artifact outcomes. The optional url-metadata.json is a separate tool-owned enrichment record. It does not rewrite acquisition provenance in capture.json. Writes stage beside the target and install with an atomic rename. --force replaces only a compatible clip-owned bundle and restores the previous bundle if installation fails.

\n

Set KB_CLIP_OUTPUT to change the default output root, or pass --output <directory> for one command. Set KB_CLIP_USER_AGENT or pass --user-agent <value> to override the default request user agent.

\n

Inspect and verify a saved bundle

\n
wordcell capture show kb/articles/example\nwordcell capture show kb/articles/example --verify-assets\nwordcell capture verify kb/articles/example\nwordcell capture verify kb/articles/example --verify-assets --json\n
\n

show reads the document, compares its v4 byte count and digest when present, and prints its Markdown as untrusted data. verify reports integrity without printing the document in its text output. Add --verify-assets to either command to read and hash every listed asset. A mismatch or an unavailable authoritative document digest returns exit status 3.

\n

The JSON forms preserve the same disclosure boundary: show --json includes the stored Markdown as explicitly untrusted inspection data, while verify --json returns only integrity metadata and issues. Verification JSON omits both the Markdown and retained source HTML so an integrity check does not accidentally disclose captured content.

\n

Schema v4 records document.path, document.bytes, and document.sha256 in capture.json. The lowercase SHA-256 digest covers the exact credential-redacted, newline-terminated Markdown bytes that Wordcell writes. It does not cover capture.json, source HTML, or assets. Each asset has a separate byte count and SHA-256 digest. Schema versions one through three report document integrity as unavailable because they do not contain the v4 document record; verify does not present that state as success.

\n

When asset verification is requested, a listed asset that is absent is an integrity mismatch rather than an operational reader crash. It produces an asset-integrity issue and exit status 3, like a digest or byte-count mismatch. Structural alias, link, confinement, and resource-budget failures remain operational errors because the reader cannot safely characterize the requested bytes.

\n

Bundle inspection keeps the canonical root open, rejects linked or aliased path ancestors and linked leaves, and rechecks root, ancestor, and file identities around every read. Asset verification is also bounded by file count, per-file bytes, aggregate bytes, and elapsed time. The SDK defaults are 1,000 files, 100 MB per file, 500 MB total, and 30 seconds; callers may raise them only to the capture contract's hard ceilings. These checks protect against hostile stored bundle structure and ordinary concurrent replacement. A malicious process with the same filesystem authority that can swap and restore path components during a system call is outside the pure-JavaScript reader's boundary; do not inspect bundles in a directory concurrently writable by an untrusted local process.

\n

Source HTML remains opt-in when you read a bundle:

\n
wordcell capture show kb/articles/example --include-source-html\n
\n

This flag has an effect only when the capture used --evidence source or --evidence all. Wordcell removes active subtrees and form state, redacts credential-shaped values, and stores the result as inert HTML. Treat the retained text as hostile data, not instructions. It can still contain private content or text intended to influence an agent.

\n

Use Git for capture history

\n

Commit retained capture bundles to Git, then compare the current Markdown document with an earlier revision:

\n
wordcell capture diff kb/articles/example\nwordcell capture diff /path/to/repository/kb/articles/example --repo /path/to/repository --ref main\nwordcell capture diff kb/articles/example --repo . --ref main --json\n
\n

The bundle must be inside the selected repository. The default reference is HEAD; --ref accepts a bounded branch, tag, or commit name, not a revision expression such as HEAD~1. Wordcell validates the work tree and ref separately, so a bad repository or ref is not mislabeled as a missing file. The command reports changed, unchanged, or missing-at-ref for the exact Markdown path and emits a bounded Git diff when it changed. It rereads and matches the capture digest after diff generation, rejecting a concurrent change instead of pairing one digest with another snapshot's diff. It does not compare the manifest, source HTML, or assets. Run wordcell capture verify separately when integrity matters. Git is the capture version history; Wordcell does not maintain a second content-history database.

\n

Track capture jobs programmatically

\n

Use the optional @hraness/wordcell/clip/jobs ledger when a service needs durable capture progress:

\n
import {\n  completeCaptureJob,\n  createCaptureJob,\n  openCaptureJobStore,\n  updateCaptureJob,\n} from "@hraness/wordcell/clip/jobs";\n\nconst store = await openCaptureJobStore("/absolute/path/to/capture-jobs");\nlet job = await createCaptureJob(store, { target: "https://example.com/article" });\njob = await updateCaptureJob(store, job.id, {\n  expectedRevision: job.revision,\n  phase: "acquiring",\n});\njob = await completeCaptureJob(store, job.id, {\n  expectedRevision: job.revision,\n  status: "complete",\n});\n
\n

Create the dedicated canonical mode-0700 store directory before opening it, and pass the returned store to every operation. The opener rejects filesystem aliases, links, and group/world access. Each job is one private, bounded UUID JSON record. Creation starts revision 1 in running and queued; updates require the expected revision, append attempts and warnings, and move phases forward. Completion records a capture status. It can also record bundle: { path, sha256 }; use capture.json's v4 document.sha256 as sha256 because the ledger does not calculate a separate bundle digest. failCaptureJob records an operational error separately from capture status. readCaptureJob and listCaptureJobs inspect retained records. A terminal completed or failed record cannot transition again, and a process crash leaves the last running record visible.

\n

The ledger redacts credential-shaped values and strips terminal controls from targets, attempts, warnings, bundle paths, and errors. A store-local, heartbeat-backed filesystem lease serializes cooperating processes across revision validation, atomic whole-record replacement, record verification, and directory synchronization; exactly one same-revision transition can succeed. There is no shared append log. Records remain until the caller applies an explicit external retention policy. The module provides no delete or automatic pruning API, and it does not replace Git as capture history. The lease is a same-host coordination mechanism, not a distributed lock for multi-host network filesystems, and a malicious same-user writer with direct store access remains outside its guarantee.

\n

Select acquisition and scope

\n
wordcell clip https://example.com/article --mode http\nwordcell clip https://example.com/application --mode browser\nwordcell clip https://example.com/post --scope page\nwordcell clip https://example.com/post --scope thread\nwordcell clip https://example.com/discussion --scope comments\n
\n

auto is the normal acquisition mode. http disables browser fallback but retains the final read-only archive lookup. browser requires rendered state and does not query Archive.today. Saved HTML can be imported without browser automation:

\n
wordcell clip https://example.com/article --html "$KB_SAVED_HTML"\nwordcell clip https://example.com/article --html - < page.html\n
\n

Default resource bounds are 30 seconds per request, process, or extraction operation; 500 scoped items; depth 16; 25 MB of HTML; 100 MB per asset; and 500 MB across assets. Browser observation also has fixed DOM and scroll ceilings. Reaching a bound is recorded and can downgrade a result to partial.

\n

Capture images, media, and evidence

\n
wordcell clip https://example.com/article --media none\nwordcell clip https://example.com/article --media images\nwordcell clip https://example.com/video --media all\nwordcell clip https://example.com/article --evidence source\nwordcell clip https://example.com/article --evidence screenshot\nwordcell clip https://example.com/article --evidence all\n
\n

Image downloads are signature-checked, content-addressed, byte-bounded, and\nrewritten to relative bundle paths. Failed images remain inert links. Video\nposters and thumbnails exposed by the page are localized without downloading\nthe full video.

\n

For YouTube, the normal capture route uses yt-dlp to retain the title,\ndescription, duration, channel, thumbnail, and one exact-language transcript\nwhen those fields are available. Full audio/video download remains opt-in:\n--media all invokes yt-dlp for accessible media, and ffmpeg may be required\nfor merging or remuxing.

\n

Source evidence is sanitized into inert HTML with credential-shaped values redacted and a deny-all content security policy. Screenshots are viewport-only pixels and are not structurally sanitized. They may contain private content or notifications, so review them before retaining or sharing a bundle.

\n

Capture a signed-in page

\n

If the page is already open, read the current tab without navigating it:

\n
wordcell clip current --browser-live\nwordcell clip current --cdp 9222\n
\n

For --browser-live, first enable Chrome's local debugging connection at chrome://inspect/#remote-debugging (Chrome 144+). If Chrome was launched with an explicit loopback debugging port, pass that numeric port to --cdp instead.

\n

To open a URL with existing browser state, select a profile name or path. Path-backed profiles run from a temporary copy, so the source profile is unchanged:

\n
wordcell clip https://example.com/member/article --browser-profile "$KB_CAPTURE_PROFILE"\n
\n

Cookie-backed HTTP capture is useful when the page does not require local storage, IndexedDB, or other browser-only state:

\n
wordcell clip https://example.com/member/article --cookie-source chrome --cookie-profile "Default"\nwordcell clip https://example.com/member/article --cookies-file "$KB_COOKIES_FILE"\n
\n

Choose at most one browser session and one cookie input. A browser session may use a separate cookie input for later asset or media downloads because attached browser state is not exported.

\n

On a Mac, reading Chrome, Arc, Brave, Chromium, or Edge cookies asks the keychain for that browser's "Safe Storage" key. The dialog names security, the macOS tool Wordcell uses to read it, so Wordcell prints a note first that says why. Enter your Mac password if asked, then choose Always Allow so macOS doesn't ask again. Wordcell never stores the key. If you deny the request, Wordcell says so and suggests --cookies-file. Safari cookies need Full Disk Access for your terminal app, because macOS doesn't ask for it. Turn it on in System Settings › Privacy & Security › Full Disk Access.

\n

Current-tab capture issues no navigation, click, form, typing, upload, or submit command. URL-based browser capture may navigate and scroll within fixed work limits, taking bounded observations as content is rendered. Both routes are ingestion-only: they do not post, like, follow, send, delete, or submit.

\n

Interpret status and counts

\n

Capture status is one of:

\n\n

For page scope, counts cover primary entries. For thread and comment scopes, counts cover replies or comments and exclude roots, quotes, ancestors, and pagination markers. Generic rendered conversations often report capturedItems: 0 because visible prose does not prove a trustworthy per-item tree.

\n

A complete or partial capture exits with status 0. Authentication, blocked, and unsupported outcomes use status 3. Argument errors use status 2, environment diagnostic failures use status 4, and operational errors use status 1. Automation should inspect the structured status and warnings rather than relying only on the process exit code.

\n

Platform routes

\n\n

Platform markup and endpoints change. Run wordcell adapters for the installed version's current claims.

\n

Backfill saved URL metadata

\n

The URL metadata command enriches every external URL record under articles/, including legacy web clips and remote PDF sources. It skips local-only PDFs. It writes one sibling url-metadata.json without changing the source Markdown or synthesizing an old capture manifest.

\n

Build the isolated Rust helper, then run the resumable backfill:

\n
wordcell url-metadata tool build\nwordcell url-metadata backfill --root .\n
\n

The helper pins MikeLuu99/searxng-rust at revision f40a00ea67a857ee996e1caba1ebab3ee7a14a47. Its crate is named metadata-search-engine-rs; it queries DuckDuckGo, Brave, Startpage, and Yahoo and combines their results. Wordcell resolves those four fixed hosts through its public-network boundary, passes only the validated addresses to the helper, disables redirects, and queries the engines serially. Wordcell validates DuckDuckGo HTML itself and limits its decoded body to 2 MiB. A challenge page or unrecognized response marks that engine as failed; only recognized result entries or an explicit no-results message count as a valid response. Empty responses from the other engines remain unverified and mark those engines as failed. A bounded global allocator caps Rust-owned response buffers and parsed data at 128 MiB. Linux also applies a 256 MiB process data ceiling. macOS rejects a lowered RLIMIT_DATA, so it retains the allocator ceiling plus the parent's subprocess deadline and output bounds. Wordcell passes requests over stdin to a private subprocess with ambient proxy and credential variables removed, bounds the subprocess deadline and output, parses the closed response itself, and does not use the upstream URL normalizer for saved identity.

\n

This operation discloses each eligible source URL as an exact search query to those search engines. Archive discovery also discloses it to Archive.today. Before either request, Wordcell resolves the source host through its public-network boundary and rejects credential-bearing URLs, private targets, and credential-shaped query parameters. Safe identity parameters such as Hacker News item?id= and YouTube watch?v= remain part of the exact query.

\n

Only a result with the same conservative URL identity can supply the selected title or description. A timestamped Archive.today result must embed that same source identity before it can be recorded. Provider failures, partial engine coverage, throttling, and absence remain explicit in the sidecar. A zero-result record is not-found only after complete attempted coverage; any failed engine or archive lookup keeps the top-level result partial.

\n

The command skips compatible sidecars by default. Use --refresh to query them again, --no-archive to omit Archive.today discovery, or --delay-ms to change the default one-second interval between outbound requests. It reads Markdown and sidecars through validated no-follow descriptors, carries the source Markdown and article-directory identities through provider work, and revalidates them under an inode-bound kernel advisory lock immediately before atomic installation. A live writer's lock cannot be stolen. The kernel releases a crashed writer's lock, and the next run safely reuses its verified lock file while UUID-scoped temporary names prevent an orphan from blocking progress. Malformed, linked, mismatched, replaced, or concurrently changed sources, locks, and sidecars fail closed.

\n", - "pdf": "

Capture PDF documents

\n

wordcell pdf converts a local PDF or a public HTTP(S) PDF into an auditable Markdown\nbundle. Remote downloads use the same DNS-pinned, private-network-denying\nacquisition boundary as web capture, and sensitive URL parameters are removed\nbefore provenance is persisted. It preserves\nthe original bytes, reconstructs native document structure, extracts embedded\nimages, and uses local OCR for text that exists only inside scans or\nscreenshots.

\n

Capture a PDF

\n
wordcell doctor\nwordcell pdf "/absolute/path/to/document.pdf" --output articles\nwordcell pdf "https://example.com/document.pdf" --output articles\nwordcell pdf "/absolute/path/to/document.pdf" --slug stable-slug --output articles\n
\n

The output mirrors a web capture without pretending the local file is a URL:

\n
articles/<slug>/\n  <slug>.md\n  capture.json\n  source.pdf\n  annotations.json  # present after a reviewed annotation pass\n  assets/\n
\n

source.pdf is byte-identical to the input. capture.json records the\noriginal filename, SHA-256, bytes, page count, bounded PDF metadata, processed\npage count, tool versions, text blocks, inferred headings, image geometry, OCR\nmethod and confidence, embedded platforms, and warnings. It never records the\ninput's original absolute path.

\n

When --annotations is used, the bundle also retains the normalized annotation\narray as annotations.json. Its path, count, byte count, and SHA-256 in\ncapture.json bind the reviewed interpretation to the saved evidence and make\nthe second pass reproducible.

\n

Pass --force only to replace a compatible PDF bundle previously created at\nthe same slug. Installation stages beside the target and commits with an\natomic rename.

\n

Text, headings, and page order

\n

Poppler supplies native text, font, emphasis, link, geometry, and page data.\nThe converter infers a body style from the document and promotes only\nwell-supported heading candidates. Uncertain structure remains a paragraph\ninstead of becoming a confident but invented hierarchy.

\n

Every emitted page is marked in the Markdown, and the manifest records\nprocessed-page and block counts. Repeated page headers and footers can be\nomitted from the readable Markdown while remaining accounted for by the source\nPDF. Font and spacing can identify a likely heading without proving its\nsemantic depth, so review peer-versus-child levels against the source.

\n

Images and OCR

\n

Every extracted image becomes a content-addressed local asset and remains\nembedded in the Markdown. A text-bearing image also receives a Markdown\ntranscription. A mixed screenshot keeps both\nrepresentations so a message, chart, photograph, code sample, or UI state is\nnot discarded merely because OCR found words.

\n

Tesseract is the local OCR fallback. Its output is bounded and records method,\nword count, and confidence. Missing OCR or an unclassified image makes the\ncapture partial but never removes the image. Screenshot-heavy captures\nshould be visually reviewed; recognizable Slack or other conversation images\ncan preserve visible platform, author, channel, timestamp, and reply metadata\nwithout inventing fields the image does not show.

\n

For a reviewed second pass, copy each image's id and asset.sha256 from the\nfirst capture.json into an annotation array. A text or mixed annotation\naccepts markdown, optional method (agent or manual), and optional\nmetadata fields for platform, contentType, channel, author,\ntimestamp, and participants. A visual annotation accepts an optional\nalt. Re-run against the same bundle with:

\n
wordcell pdf "/absolute/path/to/document.pdf" \\\n  --output articles \\\n  --annotations /tmp/pdf-image-annotations.json \\\n  --force\n
\n

IDs and hashes bind the interpretation to the exact extracted image; stale or\nmisaddressed annotations are rejected. Annotation markdown contains only the\ntranscription; the renderer supplies the image embed, metadata line, and\n“Text visible in…” heading. The normalized input is copied into the installed\nbundle, so the path passed to --annotations does not need to remain available.

\n

Local tools

\n

PDF ingestion uses the open-source Poppler command-line tools pdfinfo and\npdftohtml. Tesseract enables image text extraction. wordcell doctor reports\nwhether each route is available.

\n

Tool processes run with fixed argument shapes, bounded output, wall-time\nlimits, page and image caps, per-asset limits, and an aggregate asset budget.\nReaching any boundary is recorded and makes the result partial.

\n

Review

\n
    \n
  1. Compare source and processed page counts.
  2. \n
  3. Sample native-text, screenshot, scanned, and visual-only pages.
  4. \n
  5. Confirm heading levels match the source.
  6. \n
  7. Confirm every asset is referenced by the Markdown or manifest.
  8. \n
  9. Inspect low-confidence OCR against its source image.
  10. \n
  11. Keep later summaries and interpretations in maintained notes linked to the\ncapture.
  12. \n
\n", + "capture": "

Capture web content

\n

wordcell clip saves public and signed-in web content as an auditable Markdown bundle. It combines bounded structured adapters, HTTP extraction, browser rendering, a read-only Archive.today fallback, localized assets, and explicit completeness metadata.

\n

Check local capabilities

\n

Run the diagnostics before using local search, browser state, PDF ingestion, or video capture:

\n
wordcell doctor\nwordcell adapters\n
\n

wordcell doctor --json reports the installed runtime, QMD and static semantic-search\nprerequisites, extraction dependencies, browser support, profile display names,\nyt-dlp, ffmpeg, Poppler (pdfinfo and pdftohtml), and Tesseract. The semantic\nreport distinguishes model-free keyword readiness from SQLite and sqlite-vec\nvector prerequisites; it does not inspect or download the embedding model. The\nPoppler pair is required by wordcell pdf; Tesseract adds local OCR for scans and\nscreenshots.\nwordcell adapters --json returns the current platform capability matrix.

\n

Capture or inspect a page

\n
wordcell clip https://example.com/article\nwordcell inspect https://example.com/article\nwordcell inspect https://example.com/article --json\n
\n

The default route tries stable public structured data when available, bounded HTTP extraction, and a rendered browser when the platform or result requires one. If those routes produce no usable representation, Wordcell may make one read-only lookup for the exact URL through Archive.today's fixed archive.ph/newest/ route. An authentication, paywall, CAPTCHA, access-control, or rate-limit response disables that fallback instead of using an archive to bypass the current source's controls. Wordcell never submits the URL for archiving, retries through aliases, or lets archived HTML displace a complete or partial Hacker News or Bluesky structured capture. It validates the source through the public-network boundary before disclosure, shares one deadline across validation and provider requests, and binds every snapshot and redirect to the exact source. An archived result is rescored and always reported as partial; it keeps the original canonical URL and records the timestamped snapshot as its acquisition URL. Inspection returns the selected Markdown and capture report without writing artifacts.

\n

By default, a capture writes kb/articles/<slug>/:

\n
<slug>/\n  <slug>.md\n  capture.json\n  url-metadata.json # after an optional metadata backfill\n  assets/\n  evidence/       # only when requested\n
\n

The Markdown records source and capture metadata. capture.json records the acquisition attempts, selected extractor, scope, status, item counts, warnings, asset hashes, and requested artifact outcomes. The optional url-metadata.json is a separate tool-owned enrichment record. It does not rewrite acquisition provenance in capture.json. Writes stage beside the target and install with an atomic rename. --force replaces only a compatible clip-owned bundle and restores the previous bundle if installation fails.

\n

Set KB_CLIP_OUTPUT to change the default output root, or pass --output <directory> for one command. Set KB_CLIP_USER_AGENT or pass --user-agent <value> to override the default request user agent.

\n

Inspect and verify a saved bundle

\n
wordcell capture show kb/articles/example\nwordcell capture show kb/articles/example --verify-assets\nwordcell capture verify kb/articles/example\nwordcell capture verify kb/articles/example --verify-assets --json\n
\n

show reads the document, compares its v4 byte count and digest when present, and prints its Markdown as untrusted data. verify reports integrity without printing the document in its text output. Add --verify-assets to either command to read and hash every listed asset. A mismatch or an unavailable authoritative document digest returns exit status 3.

\n

The JSON forms preserve the same disclosure boundary: show --json includes the stored Markdown as explicitly untrusted inspection data, while verify --json returns only integrity metadata and issues. Verification JSON omits both the Markdown and retained source HTML so an integrity check does not accidentally disclose captured content.

\n

Schema v4 records document.path, document.bytes, and document.sha256 in capture.json. The lowercase SHA-256 digest covers the exact credential-redacted, newline-terminated Markdown bytes that Wordcell writes. It does not cover capture.json, source HTML, or assets. Each asset has a separate byte count and SHA-256 digest. Schema versions one through three report document integrity as unavailable because they do not contain the v4 document record; verify does not present that state as success.

\n

When asset verification is requested, a listed asset that is absent is an integrity mismatch rather than an operational reader crash. It produces an asset-integrity issue and exit status 3, like a digest or byte-count mismatch. Structural alias, link, confinement, and resource-budget failures remain operational errors because the reader cannot safely characterize the requested bytes.

\n

Bundle inspection keeps the canonical root open, rejects linked or aliased path ancestors and linked leaves, and rechecks root, ancestor, and file identities around every read. Asset verification is also bounded by file count, per-file bytes, aggregate bytes, and elapsed time. The SDK defaults are 1,000 files, 100 MB per file, 500 MB total, and 30 seconds; callers may raise them only to the capture contract's hard ceilings. These checks protect against hostile stored bundle structure and ordinary concurrent replacement. A malicious process with the same filesystem authority that can swap and restore path components during a system call is outside the pure-JavaScript reader's boundary; do not inspect bundles in a directory concurrently writable by an untrusted local process.

\n

Source HTML remains opt-in when you read a bundle:

\n
wordcell capture show kb/articles/example --include-source-html\n
\n

This flag has an effect only when the capture used --evidence source or --evidence all. Wordcell removes active subtrees and form state, redacts credential-shaped values, and stores the result as inert HTML. Treat the retained text as hostile data, not instructions. It can still contain private content or text intended to influence an agent.

\n

Use Git for capture history

\n

Commit retained capture bundles to Git, then compare the current Markdown document with an earlier revision:

\n
wordcell capture diff kb/articles/example\nwordcell capture diff /path/to/repository/kb/articles/example --repo /path/to/repository --ref main\nwordcell capture diff kb/articles/example --repo . --ref main --json\n
\n

The bundle must be inside the selected repository. The default reference is HEAD; --ref accepts a bounded branch, tag, or commit name, not a revision expression such as HEAD~1. Wordcell validates the work tree and ref separately, so a bad repository or ref is not mislabeled as a missing file. The command reports changed, unchanged, or missing-at-ref for the exact Markdown path and emits a bounded Git diff when it changed. It rereads and matches the capture digest after diff generation, rejecting a concurrent change instead of pairing one digest with another snapshot's diff. It does not compare the manifest, source HTML, or assets. Run wordcell capture verify separately when integrity matters. Git is the capture version history; Wordcell does not maintain a second content-history database.

\n

Track capture jobs programmatically

\n

Use the optional @hraness/wordcell/clip/jobs ledger when a service needs durable capture progress:

\n
import {\n  completeCaptureJob,\n  createCaptureJob,\n  openCaptureJobStore,\n  updateCaptureJob,\n} from "@hraness/wordcell/clip/jobs";\n\nconst store = await openCaptureJobStore("/absolute/path/to/capture-jobs");\nlet job = await createCaptureJob(store, { target: "https://example.com/article" });\njob = await updateCaptureJob(store, job.id, {\n  expectedRevision: job.revision,\n  phase: "acquiring",\n});\njob = await completeCaptureJob(store, job.id, {\n  expectedRevision: job.revision,\n  status: "complete",\n});\n
\n

Create the dedicated canonical mode-0700 store directory before opening it, and pass the returned store to every operation. The opener rejects filesystem aliases, links, and group/world access. Each job is one private, bounded UUID JSON record. Creation starts revision 1 in running and queued; updates require the expected revision, append attempts and warnings, and move phases forward. Completion records a capture status. It can also record bundle: { path, sha256 }; use capture.json's v4 document.sha256 as sha256 because the ledger does not calculate a separate bundle digest. failCaptureJob records an operational error separately from capture status. readCaptureJob and listCaptureJobs inspect retained records. A terminal completed or failed record cannot transition again, and a process crash leaves the last running record visible.

\n

The ledger redacts credential-shaped values and strips terminal controls from targets, attempts, warnings, bundle paths, and errors. A store-local, heartbeat-backed filesystem lease serializes cooperating processes across revision validation, atomic whole-record replacement, record verification, and directory synchronization; exactly one same-revision transition can succeed. There is no shared append log. Records remain until the caller applies an explicit external retention policy. The module provides no delete or automatic pruning API, and it does not replace Git as capture history. The lease is a same-host coordination mechanism, not a distributed lock for multi-host network filesystems, and a malicious same-user writer with direct store access remains outside its guarantee.

\n

Select acquisition and scope

\n
wordcell clip https://example.com/article --mode http\nwordcell clip https://example.com/application --mode browser\nwordcell clip https://example.com/post --scope page\nwordcell clip https://example.com/post --scope thread\nwordcell clip https://example.com/discussion --scope comments\n
\n

auto is the normal acquisition mode. http disables browser fallback but retains the final read-only archive lookup. browser requires rendered state and does not query Archive.today. Saved HTML can be imported without browser automation:

\n
wordcell clip https://example.com/article --html "$KB_SAVED_HTML"\nwordcell clip https://example.com/article --html - < page.html\n
\n

Default resource bounds are 30 seconds per request, process, or extraction operation; 500 scoped items; depth 16; 25 MB of HTML; 100 MB per asset; and 500 MB across assets. Browser observation also has fixed DOM and scroll ceilings. Reaching a bound is recorded and can downgrade a result to partial.

\n

Capture images, media, and evidence

\n
wordcell clip https://example.com/article --media none\nwordcell clip https://example.com/article --media images\nwordcell clip https://example.com/video --media all\nwordcell clip https://example.com/article --evidence source\nwordcell clip https://example.com/article --evidence screenshot\nwordcell clip https://example.com/article --evidence all\n
\n

Image downloads are signature-checked, content-addressed, byte-bounded, and\nrewritten to relative bundle paths. Failed images remain inert links. Video\nposters and thumbnails exposed by the page are localized without downloading\nthe full video.

\n

For YouTube, the normal capture route uses yt-dlp to retain the title,\ndescription, duration, channel, thumbnail, and one exact-language transcript\nwhen those fields are available. Full audio/video download remains opt-in:\n--media all invokes yt-dlp for accessible media, and ffmpeg may be required\nfor merging or remuxing.

\n

Source evidence is sanitized into inert HTML with credential-shaped values redacted and a deny-all content security policy. Screenshots are viewport-only pixels and are not structurally sanitized. They may contain private content or notifications, so review them before retaining or sharing a bundle.

\n

Capture a signed-in page

\n

If the page is already open, read the current tab without navigating it:

\n
wordcell clip current --browser-live\nwordcell clip current --cdp 9222\n
\n

For --browser-live, first enable Chrome's local debugging connection at chrome://inspect/#remote-debugging (Chrome 144+). If Chrome was launched with an explicit loopback debugging port, pass that numeric port to --cdp instead.

\n

To open a URL with existing browser state, select a profile name or path. Path-backed profiles run from a temporary copy, so the source profile is unchanged:

\n
wordcell clip https://example.com/member/article --browser-profile "$KB_CAPTURE_PROFILE"\n
\n

Cookie-backed HTTP capture is useful when the page does not require local storage, IndexedDB, or other browser-only state:

\n
wordcell clip https://example.com/member/article --cookie-source chrome --cookie-profile "Default"\nwordcell clip https://example.com/member/article --cookies-file "$KB_COOKIES_FILE"\n
\n

Choose at most one browser session and one cookie input. A browser session may use a separate cookie input for later asset or media downloads because attached browser state is not exported.

\n

On a Mac, reading Chrome, Arc, Brave, Chromium, or Edge cookies asks the keychain for that browser's "Safe Storage" key. The dialog names security, the macOS tool Wordcell uses to read it, so Wordcell prints a note first that says why. Enter your Mac password if asked, then choose Always Allow so macOS doesn't ask again. Wordcell never stores the key. If you deny the request, Wordcell says so and suggests --cookies-file. Safari cookies need Full Disk Access for your terminal app, because macOS doesn't ask for it. Turn it on in System Settings › Privacy & Security › Full Disk Access.

\n

Current-tab capture issues no navigation, click, form, typing, upload, or submit command. URL-based browser capture may navigate and scroll within fixed work limits, taking bounded observations as content is rendered. Both routes are ingestion-only: they do not post, like, follow, send, delete, or submit.

\n

Interpret status and counts

\n

Capture status is one of:

\n\n

For page scope, counts cover primary entries. For thread and comment scopes, counts cover replies or comments and exclude roots, quotes, ancestors, and pagination markers. Generic rendered conversations often report capturedItems: 0 because visible prose does not prove a trustworthy per-item tree.

\n

A complete or partial capture exits with status 0. Authentication, blocked, and unsupported outcomes use status 3. Argument errors use status 2, environment diagnostic failures use status 4, and operational errors use status 1. Automation should inspect the structured status and warnings rather than relying only on the process exit code.

\n

Platform routes

\n\n

Platform markup and endpoints change. Run wordcell adapters for the installed version's current claims.

\n

Backfill saved URL metadata

\n

The URL metadata command enriches every external URL record under articles/, including legacy web clips and remote PDF sources. It skips local-only PDFs. It writes one sibling url-metadata.json without changing the source Markdown or synthesizing an old capture manifest.

\n

Build the isolated Rust helper, then run the resumable backfill:

\n
wordcell url-metadata tool build\nwordcell url-metadata backfill --root .\n
\n

The helper pins MikeLuu99/searxng-rust at revision f40a00ea67a857ee996e1caba1ebab3ee7a14a47. Its crate is named metadata-search-engine-rs; it queries DuckDuckGo, Brave, Startpage, and Yahoo and combines their results. Wordcell resolves those four fixed hosts through its public-network boundary, passes only the validated addresses to the helper, disables redirects, and queries the engines serially. Wordcell validates DuckDuckGo HTML itself and limits its decoded body to 2 MiB. A challenge page or unrecognized response marks that engine as failed; only recognized result entries or an explicit no-results message count as a valid response. Empty responses from the other engines remain unverified and mark those engines as failed. A bounded global allocator caps Rust-owned response buffers and parsed data at 128 MiB. Linux also applies a 256 MiB process data ceiling. macOS rejects a lowered RLIMIT_DATA, so it retains the allocator ceiling plus the parent's subprocess deadline and output bounds. Wordcell passes requests over stdin to a private subprocess with ambient proxy and credential variables removed, bounds the subprocess deadline and output, parses the closed response itself, and does not use the upstream URL normalizer for saved identity.

\n

This operation discloses each eligible source URL as an exact search query to those search engines. Archive discovery also discloses it to Archive.today. Before either request, Wordcell resolves the source host through its public-network boundary and rejects credential-bearing URLs, private targets, and credential-shaped query parameters. Safe identity parameters such as Hacker News item?id= and YouTube watch?v= remain part of the exact query.

\n

Only a result with the same conservative URL identity can supply the selected title or description. A timestamped Archive.today result must embed that same source identity before it can be recorded. Provider failures, partial engine coverage, throttling, and absence remain explicit in the sidecar. A zero-result record is not-found only after complete attempted coverage; any failed engine or archive lookup keeps the top-level result partial.

\n

The command skips compatible sidecars by default. Use --refresh to query them again, --no-archive to omit Archive.today discovery, or --delay-ms to change the default one-second interval between outbound requests. It reads Markdown and sidecars through validated no-follow descriptors, carries the source Markdown and article-directory identities through provider work, and revalidates them under an inode-bound kernel advisory lock immediately before atomic installation. A live writer's lock cannot be stolen. The kernel releases a crashed writer's lock, and the next run safely reuses its verified lock file while UUID-scoped temporary names prevent an orphan from blocking progress. Malformed, linked, mismatched, replaced, or concurrently changed sources, locks, and sidecars fail closed.

\n

Observed article metadata

\n

New captures include an optional metadata envelope in capture.json. Schema 1\nkeeps each observed value, its normalized value, source and locator, the selection\nrule, and whether that claim was selected. Conflicting claims remain available.\nThe parser reads HTML title, language, standard meta and link tags, and bounded\nJSON-LD from the captured document. It does not fetch remote JSON-LD contexts.

\n

Specific article metadata takes precedence over generic webpage metadata.\nMultiple authors and alternate links can remain selected. A date without an\nexplicit timezone stays local; a calendar date is not converted into an instant.\nMetadata URLs omit query strings, fragments and credentials. They are descriptive\nevidence and must not replace the original acquisition URL. Parser limits and\nmalformed JSON-LD appear in the envelope's warnings. Older bundles can omit the\nenvelope.

\n

@hraness/wordcell/capture exports extractArticleMetadata and\nparseArticleMetadata; the bundle reader returns this evidence when present.

\n", + "pdf": "

Capture PDF documents

\n

wordcell pdf converts a local PDF or a public HTTP(S) PDF into an auditable Markdown\nbundle. Remote downloads use the same DNS-pinned, private-network-denying\nacquisition boundary as web capture, and sensitive URL parameters are removed\nbefore provenance is persisted. It preserves\nthe original bytes, reconstructs native document structure, extracts embedded\nimages, and uses local OCR for text that exists only inside scans or\nscreenshots.

\n

Capture a PDF

\n
wordcell doctor\nwordcell pdf "/absolute/path/to/document.pdf" --output articles\nwordcell pdf "https://example.com/document.pdf" --output articles\nwordcell pdf "/absolute/path/to/document.pdf" --slug stable-slug --output articles\n
\n

The output mirrors a web capture without pretending the local file is a URL:

\n
articles/<slug>/\n  <slug>.md\n  capture.json\n  source.pdf\n  annotations.json  # present after a reviewed annotation pass\n  assets/\n
\n

source.pdf is byte-identical to the input. capture.json records the\noriginal filename, SHA-256, bytes, page count, bounded PDF metadata, processed\npage count, tool versions, text blocks, inferred headings, image geometry, OCR\nmethod and confidence, embedded platforms, and warnings. It never records the\ninput's original absolute path.

\n

When --annotations is used, the bundle also retains the normalized annotation\narray as annotations.json. Its path, count, byte count, and SHA-256 in\ncapture.json bind the reviewed interpretation to the saved evidence and make\nthe second pass reproducible.

\n

Pass --force only to replace a compatible PDF bundle previously created at\nthe same slug. Installation stages beside the target and commits with an\natomic rename.

\n

Text, headings, and page order

\n

Poppler supplies native text, font, emphasis, link, geometry, and page data.\nThe converter infers a body style from the document and promotes only\nwell-supported heading candidates. Uncertain structure remains a paragraph\ninstead of becoming a confident but invented hierarchy.

\n

Every emitted page is marked in the Markdown, and the manifest records\nprocessed-page and block counts. Repeated page headers and footers can be\nomitted from the readable Markdown while remaining accounted for by the source\nPDF. Font and spacing can identify a likely heading without proving its\nsemantic depth, so review peer-versus-child levels against the source.

\n

Images and OCR

\n

Every extracted image becomes a content-addressed local asset and remains\nembedded in the Markdown. A text-bearing image also receives a Markdown\ntranscription. A mixed screenshot keeps both\nrepresentations so a message, chart, photograph, code sample, or UI state is\nnot discarded merely because OCR found words.

\n

Tesseract is the local OCR fallback. Its output is bounded and records method,\nword count, and confidence. Missing OCR or an unclassified image makes the\ncapture partial but never removes the image. Screenshot-heavy captures\nshould be visually reviewed; recognizable Slack or other conversation images\ncan preserve visible platform, author, channel, timestamp, and reply metadata\nwithout inventing fields the image does not show.

\n

For a reviewed second pass, copy each image's id and asset.sha256 from the\nfirst capture.json into an annotation array. A text or mixed annotation\naccepts markdown, optional method (agent or manual), and optional\nmetadata fields for platform, contentType, channel, author,\ntimestamp, and participants. A visual annotation accepts an optional\nalt. Re-run against the same bundle with:

\n
wordcell pdf "/absolute/path/to/document.pdf" \\\n  --output articles \\\n  --annotations /tmp/pdf-image-annotations.json \\\n  --force\n
\n

IDs and hashes bind the interpretation to the exact extracted image; stale or\nmisaddressed annotations are rejected. Annotation markdown contains only the\ntranscription; the renderer supplies the image embed, metadata line, and\n“Text visible in…” heading. The normalized input is copied into the installed\nbundle, so the path passed to --annotations does not need to remain available.

\n

Local tools

\n

PDF ingestion uses the open-source Poppler command-line tools pdfinfo and\npdftohtml. Tesseract enables image text extraction. wordcell doctor reports\nwhether each route is available.

\n

Tool processes run with fixed argument shapes, bounded output, wall-time\nlimits, page and image caps, per-asset limits, and an aggregate asset budget.\nReaching any boundary is recorded and makes the result partial.

\n

Review

\n
    \n
  1. Compare source and processed page counts.
  2. \n
  3. Sample native-text, screenshot, scanned, and visual-only pages.
  4. \n
  5. Confirm heading levels match the source.
  6. \n
  7. Confirm every asset is referenced by the Markdown or manifest.
  8. \n
  9. Inspect low-confidence OCR against its source image.
  10. \n
  11. Keep later summaries and interpretations in maintained notes linked to the\ncapture.
  12. \n
\n

Layout quality and verification

\n

Separated columns are read in sequence before lines are merged. Compact aligned\ntables with an explicit header and numeric cells become Markdown tables. Basic\nraised digits remain superscripts. Complex formulas, ambiguous tables and\nuncertain layout produce page-specific quality issues and a partial capture;\nthe original PDF remains the reference. Repeated headers are removed only near\npage margins. These rules do not promise complete layout reconstruction.

\n

New bundles bind the Markdown bytes as well as the original PDF and images.\nUse the public SDK to inspect or verify an existing bundle:

\n
import { verifyPdfCaptureBundle } from "@hraness/wordcell/pdf";\nconst result = await verifyPdfCaptureBundle("/absolute/path/to/saved-document");\nif (!result.ok) throw new Error("PDF bundle verification failed");\n
\n

inspectPdfCaptureBundle returns the saved text and integrity results.\nverifyPdfCaptureBundle checks bounded, confined files and page/asset references.\nLegacy bundles without a Markdown digest report unavailable integrity rather\nthan verified content. PdfCaptureError.code provides stable failure categories;\nunknown failures remain unknown, and password errors require a recognized tool\ndiagnostic.

\n

A host with its own process supervisor can inject runTool and call\nrunPdfToolCommand({ ...command, processGroup: "inherit" }). That host owns\ntermination and descendant cleanup for the surrounding group. Standalone\ncommands keep the default separate process group.

\n", "publish": "

Publish a static site

\n

wordcell publish turns selected Markdown notes into a static website with\nsearch, backlinks, and a graph. Build it locally, inspect the result, then copy\nthe files to your preferred host. Publishing does not upload your vault or\nrequire an account, database, model, or API key.

\n

The HTML pages work without JavaScript. Serve the directory over HTTP to use\nthe bundled browser-local search and graph; opening HTML files directly is\nsuitable for reading pages, but browser restrictions can block the reader.

\n

Preview and publish a slice

\n

Choose notes by exact path, directory, glob, metadata, or a graph neighborhood.\nThis example combines a directory tree with one note from elsewhere in the\nvault, then removes drafts:

\n
wordcell publish --root kb --out site \\\n  --include-glob 'guides/**/*.md' --include notes/getting-started \\\n  --exclude drafts --dry-run --json\n
\n

The dry run builds the same artifact in memory and leaves the output directory\nuntouched. Its JSON report includes counts, a source digest, and up to 20\nselected note ids. The digest covers selected Markdown; it does not cover\nattachment bytes. Review those ids, counts, and referenced assets before writing. Repeat the same\nselectors without --dry-run to build the site:

\n
wordcell publish --root kb --out site \\\n  --include-glob 'guides/**/*.md' --include notes/getting-started \\\n  --exclude drafts\nwordcell serve --root site --port 8080\n
\n

Open http://127.0.0.1:8080 to inspect the pages. Nothing is publicly hosted\nuntil you upload the generated directory. When rebuilding an existing site,\nadd --force to replace that output directory. Keep it separate from your\nsource vault: neither directory may contain the other. An empty selection\nreturns a zero-count dry-run report, but writing it fails so a mistyped selector\ncannot replace an existing site. Wordcell writes the complete replacement\ninto a sibling staging directory first. A write failure leaves the previous\nsite in place. Promotion moves the previous site to a temporary backup, then\nmoves the staged site into place. These are two renames, so a running server can\nbriefly see no output directory. If promotion fails, Wordcell restores the\nbackup; if restoration also fails, the error names the retained backup path.\nThe backup is removed only after successful promotion.

\n

For agent workflows, --json returns the report without note bodies, HTML,\nsearch postings, or file contents. --list-limit 0 returns counts and the source\ndigest only; a value from zero through 1,000 changes the maximum listed ids.\nThe ids also have a combined 16 KiB byte budget, and selection.truncated\nreports when the list is incomplete. The bound changes only the report, never\nwhich notes publish. Read the generated catalog.json when you need the full\npublished list. An SDK caller can reuse a scanned vault with projectVault.

\n

Choose what publishes

\n
wordcell publish --root kb --out site/docs --include docs\nwordcell publish --root kb --out site/public --tag public --exclude drafts\nwordcell publish --root kb --out site/guides \\\n  --include-glob 'guides/**/*.md' --exclude-glob '**/draft-*'\nwordcell publish --root kb --out site/topic \\\n  --from notes/topic-seed --depth 2 --direction both\nwordcell publish --root kb --out site/concepts --where type=concept --has relations\n
\n\n

Review selected prose and attachments before hosting. Selection controls which\nnotes enter the site; it does not redact text inside them. A selected note can\nstill mention a private name, contain sensitive metadata in its search fields,\nor reference an attachment you intended to keep private. Exclusion selectors\napply to notes, not attachment files. Only validated local attachments directly\nreferenced by selected notes are copied.

\n

Links and typed relationships to excluded notes are omitted from structured\nnavigation. An authored link label can remain in the selected note's rendered\nprose. Public manifests record selector counts without storing raw include,\nexclude, filter, or seed values.

\n

Set the site identity

\n
wordcell publish --root kb --out site --title "Team Handbook" \\\n  --description "Public engineering notes" --base-path /handbook/ \\\n  --base-url https://docs.example.com\n
\n

--base-path sets the URL prefix (default /). --base-url adds a\nsitemap.xml; omit it when the site has no canonical origin. --noindex\nemits a robots disallow and a noindex marker on every page. These are indexing\nrequests, not access controls. --deterministic omits the generation timestamp\nso identical inputs and options produce byte-identical artifacts.

\n

The emitted artifact

\n
site/\n  index.html              # front door: the vault index note, or a catalog page\n  index.json              # the index note payload, when one published\n  manifest.json           # hraness.wordcell.site.v1 manifest\n  catalog.json            # every note: slug, title, type, tags\n  graph.json              # selection-scoped link and relation edges\n  graph/index.html        # interactive map of the published graph\n  n/<slug>/index.html     # prerendered note pages with backlink panels\n  n/<slug>.json           # hydrated note payloads for richer readers\n  index/docs.json         # document table with normalized eager fields\n  index/terms.json        # sorted term dictionary for prefix lookup\n  index/c/<hh>.json       # content postings shards, only for large selections\n  assets/<hash>.<ext>     # referenced attachments, content-addressed\n  reader/reader.js|.css   # the bundled reader: search overlay and graph map\n  reader/theme.js         # synchronous appearance bootstrap (see below)\n  404.html, robots.txt, sitemap.xml\n
\n

Reading the site

\n

Every page ships the same chrome: a header with the site title, a Graph link,\nand a Search button; a sidebar tree of the published notes; and a footer.\nThe tree is derived from slug paths. Directories sort before notes at each\nlevel, groups open along the path to the current page, and oversized branches\nfold into bounded +N more links back to the catalog. Everything is\nprerendered HTML: navigation works with JavaScript disabled.

\n

Note pages add breadcrumbs above the article and an On this page table of\ncontents beside it, generated from the same heading anchors the Markdown\nrenderer emits. The backlink, relation, and referenced-by panels still sit in\nthe article aside.

\n

The graph page maps graph.json onto a canvas: nodes sized by degree, links\nas solid edges and typed relations as dashed ones. Drag to pan, scroll or\npinch to zoom, hover a node to light its neighborhood, and click a node to\nopen the note. graph/#n=<slug> deep-links to a focused node. The layout is a\nseeded deterministic force layout, so every visitor sees the same map. Sites\nover the contract's node limit publish the same page with the static note\nindex instead of the live canvas, and the prerendered index below the map\nkeeps the page useful without JavaScript at any size.

\n

Footnotes

\n

Use [^label] in prose and a top-level [^label]: Source or explanation\ndefinition to publish a numbered footnote. Each citation links to its footnote;\neach footnote links back to every place that cites it. These links work without\nJavaScript. Definitions can contain ordinary links and inline formatting.\nIndent continuation lines by four spaces or one tab. Nested footnote references\ninside definitions remain literal, and supported explicit links take precedence over\nfootnote syntax in their labels.

\n
The measured result is consistent with the earlier study.[^study]\n\n[^study]: [Read the study](https://example.com/study).\n
\n

Labels allow Unicode letters, marks, and numbers plus ._:-; they are\ncase-sensitive and normalized to Unicode NFC. Keep labels within\n128 UTF-8 bytes and definitions within 32 lines and 8,192 UTF-8 bytes. A page\nsupports 256 definitions and 2,048 linked citations. Missing, duplicated,\ninvalid, and over-limit references remain visibly unresolved; their authored\ndefinitions remain visible. Footnote syntax in code or comments stays inert.

\n

Appearance

\n

The reader supports the shared Hraness palette contract: the wordcell\nneutral palette plus Catppuccin, Gruvbox, Rosé Pine, and Tokyo Night, each\nin light and dark. A small classic script, reader/theme.js, loads\nsynchronously in every page head, ahead of the stylesheet, and applies\nthe stored preference to data-palette and data-theme on the document\nelement before first paint, so a saved palette never flashes the default.\nThe preference lives under hraness-design-palette-v1 in the origin's\nlocal storage, the same key design-kit applications use, so a reader\nhosted beside one honors the visitor's existing choice; system mode\nfollows the operating system and storage events synchronize other tabs.\nThe deferred reader adds a single appearance menu as the rightmost header\naction — radio groups for mode and palette — and the graph map repaints\non every applied change. Without JavaScript, or with no stored preference,\npages fall back to the neutral palette under prefers-color-scheme.

\n

Every JSON file carries an explicit format identifier and parses under a\nbounded unknown-value contract; parsers reject unexpected keys, oversize\nfields, and wrong formats. manifest.json records the generator identity and\nversion, selector counts, the source digest, artifact paths, search\nmode, counts, and any truncated aspects. In --deterministic mode the manifest\ncarries no timestamp and the whole directory is reproducible.

\n

Browser-local search

\n

The document table eagerly stores normalized title, alias, path, tag, and\nmetadata fields so in-page matching reproduces the exact lane's substring\nsemantics. Note content is indexed inline while the selection fits the\ncontract's aggregate budget; larger selections store sharded postings that the\nreader fetches lazily by FNV-1a term hash. --no-index-content disables content\nindexing entirely, leaving field-only search. Query, prefix-expansion, result,\nand hydration bounds are fixed by the contract so a hostile or oversized\nartifact cannot exhaust the browser.

\n

The overlay opens from the Search button or the / key. Alongside free\ntext, field filters narrow the candidate set:

\n\n

Filters combine with free text. tag:public path:docs migration searches\n"migration" inside public notes under docs/ — and a filter-only query lists\neverything that matches. Unknown or malformed name: tokens stay in the\nfree-text query. Matched terms render with <mark> highlighting built from\nDOM text nodes, never injected HTML.

\n

Search excerpts show readable article text and link labels. Resolved footnote\nmarkers and Markdown formatting are omitted; code, escaped text, and unresolved\nreferences remain literal. The full Markdown is interpreted before excerpts are\nclipped, so a definition later in the note still resolves its citation. Snippets\nkeep complete Unicode characters within their byte limit.

\n

For custom readers, docs.json stores the eager preview in p and the inline\nsearch basis in x. Hydrated note text holds bounded readable content,\nnormalized to NFC and lowercase for snippet matching. Use the document table\nand content postings for ranking; hydration supplies display text. Existing\npublished artifacts keep their original excerpts until republished.

\n

Safety

\n

Raw HTML never passes through the renderer: every text span and attribute is\nentity-escaped, only the controlled Markdown grammar emits markup, and external\nURLs are restricted to http, https, and mailto. Pages ship a strict\ndefault-src 'self' Content-Security-Policy, so no remote image, script, or\nframe can load. The output directory and vault must be separate: neither may contain the\nother, including through symlinked parent directories. The output directory\nitself cannot be a symlink, and every written path is confined to --out.\nGenerated JSON must pass the reader contracts before any output is replaced.\nEvery generated filename component must fit 255 UTF-8 bytes, and a generated\nfile cannot also serve as a directory. A conflicting or oversized filename\nproduces a rename instruction before an existing site is touched.\nAn oversized title or too many aliases or tags produces a named validation\nerror; shorten that field or narrow the selection and publish again.

\n

Host the result

\n

The site is static files. Sync the output directory to any object store or\nstatic host — no adapter is required or bundled:

\n
aws s3 sync site/ s3://bucket/handbook/ --delete\nrclone sync site/ remote:handbook\n
\n

Serve under a subpath by passing --base-path at publish time. For a local\npreview, wordcell serve binds a static file server to the output directory:

\n
wordcell serve --root site --port 8080\n
\n

The server defaults to the loopback interface (--host overrides), maps\ndirectories to index.html, returns the published 404.html for missing\npaths, confines requests and symlinks to the root, and answers GET and\nHEAD only. Production hosting stays with the object store or CDN. Any other static file\nserver works for previews too:

\n
python3 -m http.server --directory site 8080\n
\n

Host on wordcell.io

\n

wordcell.io also publishes a vault for you: the API runs the same projection\nserver-side and serves the emitted artifact at https://wordcell.io/p/<key8>/<slug>/.\nThis path exists for agents and machines that cannot hold files or deploy a\nstatic host; the CLI remains the primary publication path, and the artifact\ncontract is identical either way.

\n
curl -s -X POST https://wordcell.io/api/v1/tokens -d '{"label":"my agent"}'\n# → { "ok": true, "token": "wc_pub_…", "key8": "…", "urlBase": "…/p/<key8>/" }\n\ncurl -s -X PUT https://wordcell.io/api/v1/sites/handbook \\\n  -H "Authorization: Bearer wc_pub_…" -H "content-type: application/json" -d '{\n    "operation": {\n      "contract": "hraness.wordcell.hosted-operation.v1",\n      "id": "32d708ac-782d-4ca0-8c34-3e89ad0ed02a",\n      "expectedRevision": 0\n    },\n    "title": "Team Handbook",\n    "files": {\n      "index.md": "# Handbook\\n\\nStart with [[onboarding]].\\n",\n      "onboarding.md": "# Onboarding\\n\\nWelcome.\\n",\n      "assets/logo.png": {"upload": "<id from POST /api/v1/uploads>"}\n    }\n  }'\n# → { "ok": true, "site": { "url": "https://wordcell.io/p/<key8>/handbook/", … } }\n
\n\n

GET /api/v1/openapi.json returns the OpenAPI 3.1 description;\nGET /api/v1/health reports service and storage health. MCP clients can use\nthe streamable-HTTP endpoint POST /api/v1/mcp instead of REST: it exposes\ncreate_token, get_site, publish_site, list_sites, and delete_site tools that\ndispatch to the identical route logic — send the wc_pub_ Bearer token as on\nREST, and create_token stays unauthenticated so an MCP-only client can\nonboard itself. The storage layer is\na private Cloudflare R2 bucket behind the wordcell-sites worker — object\nreads and writes are HMAC-signed, and the public /p/ path resolves the site\nhead to immutable artifact bytes with directory-index and 404.html\nsemantics identical to wordcell serve.

\n

Hosted directory pages use a trailing slash so their relative navigation and\nreader assets stay inside the edition. A slashless request redirects only\nafter the Worker finds that directory's index; files and missing paths do not\nacquire a slash. The site proxy preserves /p/ paths and keeps other site\npages on their usual slashless URLs.

\n

When introducing this routing policy, deploy the site first and verify that a\nhosted directory URL returns its published HTML and keeps its slash, then deploy\nthe Worker redirect. The Vercel configuration has a separate trailing-slash\nrewrite before the slashless rule; both must forward the original path form.\nVerify root and nested published pages on the preview deployment before merging.\nThe local Next server does not exercise Vercel's external rewrites. Reversing\nthat order creates a redirect loop with the older site. Roll back the Worker\nbefore restoring the older site policy. Existing stored artifacts need no rewrite.

\n

Programmatic use

\n

publishVault drives the filesystem pipeline; projectVault projects an\nalready-scanned snapshot through an injectable PublishIo seam, so tests and\nalternative hosts can resolve and read assets without disk access:

\n
import { publishVault } from "@hraness/wordcell/publish";\n\nconst preview = await publishVault({\n  root: "kb",\n  out: "site",\n  selection: { includeGlobs: ["guides/**/*.md"], includes: ["notes/getting-started"] },\n  dryRun: true,\n  listLimit: 10,\n});\nconsole.log(preview.report); // counts, selected ids, and digest; no note bodies\n
\n

parseSiteManifestV1 and WORDCELL_SITE_LIMITS_V1 are available from\n@hraness/wordcell/publish-model. The constants fix every artifact bound: note,\nasset, term, shard, preview, inline-text, payload-text, and hydration caps.

\n", "portfolio": "

Search several vaults together

\n

Portfolio federation searches and audits selected Markdown vaults without combining their files or indexes. A strict registry gives each vault a stable logical identity and maps that identity to a checkout under one workspace. Each vault keeps its own Markdown, local graph, optional QMD index, and Git history.

\n

The registry discovers vaults. An explicit selection determines which vaults a command or SDK session may open.

\n

Define a v1 registry

\n

A v1 registry is a JSON file with the contract hraness.kb-portfolio/v1 and schemaVersion 1:

\n
{\n  "contract": "hraness.kb-portfolio/v1",\n  "schemaVersion": 1,\n  "vaults": [\n    {\n      "owner": "example",\n      "id": "product-history",\n      "repository": "example/product-history",\n      "checkout": "history-vault",\n      "root": "kb",\n      "role": "repository",\n      "visibility": "organization",\n      "defaultRef": "main",\n      "parserVersion": 1\n    },\n    {\n      "owner": "hraness",\n      "id": "legacy-notes",\n      "repository": "hraness/legacy-notes",\n      "checkout": "legacy-notes",\n      "root": "kb",\n      "role": "archive",\n      "visibility": "private",\n      "parserVersion": 1\n    },\n    {\n      "owner": "personal",\n      "id": "tiff",\n      "repository": "personal/tiff",\n      "checkout": "tiff",\n      "root": "kb",\n      "role": "portfolio",\n      "visibility": "personal",\n      "parserVersion": 1\n    }\n  ],\n  "authorityGroups": [\n    {\n      "id": "product-history",\n      "members": [\n        "example/product-history",\n        "hraness/legacy-notes"\n      ],\n      "state": "unresolved",\n      "protected": true,\n      "reason": "An owner must choose the maintained authority."\n    }\n  ]\n}\n
\n

The vault key is the explicit owner/id pair. It remains example/product-history even when the local checkout is named history-vault. Do not derive a vault key from checkout, root, or a Git remote.

\n

The other fields have separate purposes:

\n\n

The parser rejects unknown properties and unsafe paths. A registry is limited to 1 MiB, 128 vaults, and 64 authority groups. One operation may select at most 32 vaults. --shared and --all reject a registry-derived selection above that bound; they never truncate it silently.

\n

Select vaults explicitly

\n

The SDK requires a nonempty authorizedVaults list. The CLI offers three selection forms:

\n\n

Use --vault for private and personal entries. Use --all only when the process is permitted to read every configured vault.

\n

After selection, the portfolio core resolves, opens, scans, and reports only selected vaults. Results do not expose names, counts, paths, or failures from unselected vaults. The CLI must still read and validate the registry to compute --shared or --all, so access to registry metadata is a separate prerequisite. It passes that exact immutable registry snapshot into the operation; authorization and checkout resolution cannot observe two different registry versions.

\n

Visibility is a registry classification. Operating-system permissions and workspace confinement remain the access boundary. A registry entry cannot give the process access to a checkout it cannot already read.

\n

Search selected vaults

\n

Search the shared vaults under one workspace:

\n
wordcell portfolio search "durable agent memory" \\\n  --registry ./kb-portfolio.json \\\n  --workspace /srv/knowledge \\\n  --shared \\\n  --mode hybrid \\\n  --limit 10\n
\n

Select private or personal vaults by exact key:

\n
wordcell portfolio search "release rationale" \\\n  --registry ./kb-portfolio.json \\\n  --workspace /srv/knowledge \\\n  --vault example/product-history \\\n  --vault hraness/legacy-notes \\\n  --mode exact\n
\n

The supported modes are hybrid, exact, keyword, and semantic. Add --json to inspect per-vault diagnostics and local retrieval evidence.

\n

A query that is a canonical qualified URI routes directly to its named selected vault and uses exact retrieval:

\n
wordcell portfolio search "kb://example/product-history/018f4b20-7c95-7af2-a11f-89011baf1137" \\\n  --registry ./kb-portfolio.json \\\n  --workspace /srv/knowledge \\\n  --vault example/product-history\n
\n

If that vault or document is outside the selection, the search does not reveal whether it exists elsewhere.

\n

Failure policy

\n

Portfolio search uses the partial failure policy by default. A selected vault that cannot resolve, open, or complete its search produces a selected-vault diagnostic. Results from available selected vaults remain usable, and the result has partial: true.

\n

Add --require-all when every selected vault must resolve, open, and execute its search:

\n
wordcell portfolio search "release rationale" \\\n  --registry ./kb-portfolio.json \\\n  --workspace /srv/knowledge \\\n  --shared \\\n  --require-all\n
\n

--require-all does not turn an optional QMD or Git provenance degradation into a fatal error. Inspect partial and the lane diagnostics when completeness matters.

\n

Deterministic federation order

\n

Each vault performs its own exact or QMD-backed retrieval. The federation layer keeps QMD scores inside that vault's evidence because scores from separate QMD databases are not comparable.

\n

The global merge uses this order:

\n
    \n
  1. Exact identity matches precede other matches.
  2. \n
  3. A reciprocal score derived from the result's local rank orders the remaining candidates.
  4. \n
  5. The stable vault key breaks a tie.
  6. \n
  7. The qualified identity or current path breaks the final tie.
  8. \n
\n

The returned portfolio score describes this reciprocal local-rank merge. It is not a probability or a QMD similarity score. Raw local evidence remains available under the hit's local and evidence fields.

\n

Graph traversal remains vault-local. Portfolio search does not infer a cross-vault neighborhood or use graph context as a relevance boost.

\n

Give notes stable identities

\n

Use document_id for an identity that survives a path change:

\n
---\ndocument_id: 018f4b20-7c95-7af2-a11f-89011baf1137\ntype: concept\ntitle: Durable agent memory\n---\n\n# Durable agent memory\n
\n

document_id is a lowercase ASCII ID of at most 128 bytes. It may contain letters, digits, dots, underscores, and hyphens. Ordinary notes created through wordcell note create or createNote receive a UUID v4 when the caller does not provide an ID.

\n

An idempotent create preserves an existing valid ID. It leaves a legacy note without an ID unchanged, and it rejects an explicit ID that conflicts with the existing note. Add IDs to legacy notes through a reviewed Markdown edit rather than treating an idempotent create as a migration.

\n

A stable portfolio URI combines the logical vault identity and the authored document ID:

\n
kb://owner/vault/document_id\n
\n

For example:

\n
kb://example/product-history/018f4b20-7c95-7af2-a11f-89011baf1137\n
\n

The URI is byte-canonical. Uppercase aliases, percent-encoded components, path traversal, and extra path segments are rejected. A document_id must be unique within its vault. The same ID in two different vaults produces two different qualified URIs.

\n

Notes without a valid document_id remain searchable through a tagged legacy-path identity. A legacy path is not reported as stable and cannot be used for a qualified read.

\n

Author cross-vault relations

\n

A typed relation may target a canonical qualified URI:

\n
relations:\n  supports:\n    - kb://example/product-history/018f4b20-7c95-7af2-a11f-89011baf1137\n
\n

The source note still owns the assertion. The local graph records it as an external authored relation instead of resolving it to a local note ID.

\n

Portfolio audit verifies the target against valid stable IDs in the explicitly selected, successfully audited vaults. It reports external-relation-unavailable when the target is not available in that selection. This diagnostic does not claim that the target is absent from an unselected vault.

\n

Cross-vault relations do not create inferred backlinks, reciprocal assertions, or cross-vault graph traversal. Author each assertion in Markdown where its source belongs.

\n

Audit a portfolio

\n

Run a report-only audit across the shared selection:

\n
wordcell portfolio audit \\\n  --registry ./kb-portfolio.json \\\n  --workspace /srv/knowledge \\\n  --shared\n
\n

Audit exact vaults when the review includes private material:

\n
wordcell portfolio audit \\\n  --registry ./kb-portfolio.json \\\n  --workspace /srv/knowledge \\\n  --vault example/product-history \\\n  --vault hraness/legacy-notes \\\n  --strict\n
\n

The default command exits successfully after producing the report, even when the report contains issues. --strict exits with status 3 when the report contains at least one error or is truncated. Warnings and advisories do not trigger that exit status. Use --json for the complete bounded report; its ok field is also false for an error or truncation.

\n

Audit checks include:

\n\n

Missing IDs are warnings in portfolio and repository vaults. They are advisories in archive, template, and sample vaults. Invalid or duplicate IDs are errors. Exact duplicate content is a warning when an active vault is involved and an advisory otherwise.

\n

An audit is partial when a selected vault is unavailable or the report is truncated. Ordinary findings do not by themselves make the report partial. The default issue limit is 500, and the maximum is 5,000. Severity counts include every discovered issue even when only a bounded severity-preserving sample is materialized. --strict fails closed when any error is counted or when the report is truncated.

\n

Audit is read-only. It does not assign IDs, repair links, update a catalog, deduplicate notes, move files, or choose an authority.

\n

Record authority decisions

\n

An authority group records a reviewed decision state among registry members. Use state: "unresolved" while an owner must still choose. An unresolved group cannot declare canonical.

\n

After review, change the state and name one member:

\n
{\n  "id": "product-history",\n  "members": [\n    "example/product-history",\n    "hraness/legacy-notes"\n  ],\n  "state": "resolved",\n  "canonical": "example/product-history",\n  "reason": "The maintained repository owns current product history."\n}\n
\n

protected: true marks a group whose duplicate material must remain visible during review. It does not suppress duplicate findings, grant access, or change search rank.

\n

Use this migration sequence:

\n
    \n
  1. Register each real vault with its current stable logical owner/id identity.
  2. \n
  3. Mark unresolved authority groups before moving or deleting content.
  4. \n
  5. Run a selected audit and review root, identity, relation, and duplicate findings.
  6. \n
  7. Add document_id to maintained legacy notes through reviewed Markdown edits. Preserve an ID when a note moves.
  8. \n
  9. Replace cross-vault path assumptions with reviewed kb:// relation targets.
  10. \n
  11. Resolve an authority group only after its owner selects a canonical member and records the reason.
  12. \n
  13. Enable --strict in automation after expected errors have been resolved.
  14. \n
\n

Keep protected duplicate sources until their owner approves a consolidation. The registry and audit surface an authority conflict; they do not settle it.

\n

Keep search rules opt-in

\n

Search rules are a strict configuration with schemaVersion: 1, named leading aliases, and priority rules. Configure them through OpenKnowledgeBaseOptions.searchRules, or through OpenKnowledgePortfolioOptions.knowledgeBase.searchRules for each local vault session. The CLI reads the same bounded JSON contract from --rules <file>.

\n

For example, a consumer-owned search-rules.json can define one constrained alias and one opt-in maintained-note tier:

\n
{\n  "schemaVersion": 1,\n  "aliases": {\n    "active-plans": {\n      "mode": "hybrid",\n      "filters": [\n        {\n          "kind": "one-of",\n          "path": "status",\n          "values": ["proposed", "accepted", "in-progress"]\n        }\n      ]\n    }\n  },\n  "priorityRules": [\n    {\n      "id": "maintained-notes",\n      "tier": 1,\n      "pathPrefix": "notes/"\n    }\n  ]\n}\n
\n

The full alias fields are query, mode, filters, tags, and repositoryScopes. A priority rule needs a canonical id, a tier from 1 through 32, and at least one of pathPrefix, tagsAll, repositoryScope, metadata, or vaultId. Unknown fields and malformed predicates are rejected.

\n

A leading alias expands only when it is the first query token, such as @active context. Alias filters, tags, and repository scopes add constraints. Caller-provided constraints are preserved.

\n

Priority rules may match pathPrefix, tagsAll, repositoryScope, metadata, or vaultId. Conditions on one rule are combined with AND. Smaller positive tier values are preferred. Exact identity matches remain ahead of every tier, and ties retain the original relevance order.

\n

Priority ordering applies only when a search request sets:

\n
ordering: "priority-then-relevance"\n
\n

The default is ordering: "relevance", which leaves configured priority rules inactive. A single-vault KnowledgeBaseSearchResult includes a trace when at least one selected hit matches a priority rule, whether or not that match changes the final ordering. Search rules do not create authority, permissions, graph edges, or Markdown edits.

\n

On the CLI, --rules enables leading aliases. Add --priority to request priority ordering:

\n
wordcell search "@active parser" --root ./kb \\\n  --rules ./search-rules.json \\\n  --priority\n\nwordcell portfolio search "@active parser" \\\n  --registry ./kb-portfolio.json \\\n  --workspace /srv/knowledge \\\n  --shared \\\n  --rules ./search-rules.json \\\n  --priority\n
\n

--priority without --rules is rejected. A rules file may contain at most 128 KiB of configured text and is parsed before a vault scan.

\n

Within a portfolio, an explicitly requested priority pass may reorder results inside each vault. The portfolio result reflects the resulting local ranks and retains each selected vault's rule trace in its diagnostic. Federation still combines those ranks by exact identity, reciprocal local rank, stable vault key, and identity. It never compares raw QMD scores across vaults.

\n

A vaultId condition applies only when the rules caller supplies logical vault context. openKnowledgePortfolio injects each selected registry key into its local priority pass. A standalone openKnowledgeBase caller can set vaultId explicitly. A vault condition is not a global authority weight.

\n

The CLI and SDK use the same opt-in ordering. Keep the rules file under review beside the consumer configuration that owns its retrieval policy.

\n

Markdown remains authoritative

\n

The registry maps stable logical identities to selected directories. It does not contain note facts or generated graph edges. Authority groups record a review state without rewriting either vault.

\n

Each vault's links and typed relations are derived from its current Markdown. Cross-vault relations are also authored in source-note frontmatter. Audit builds a temporary selected view, reports its findings, and discards that view.

\n

QMD remains an optional per-vault retrieval cache. It can be deleted and rebuilt from Markdown. Federation does not create a central vector index, aggregate graph database, reciprocal edge file, or generated fact store. Git HEAD and content hashes provide provenance; they do not replace authored content.

\n

Threat boundaries

\n

Treat the registry, vault files, search queries, and returned note text as untrusted input.

\n\n

These boundaries keep federation read-only and selected. They do not make a shared registry an access-control system or make untrusted Markdown safe to execute.

\n", "sync": "

Sync a vault with Git

\n

Wordcell has no sync service. A vault is a folder of Markdown files, so a Git\nremote can carry it between machines, and each machine rebuilds its own search\nindex from those files. This guide puts a vault in a private repository, pulls\nit before you work, commits and pushes it on a schedule, and resolves the\nconflicts Git cannot merge.

\n

Choose where the vault lives

\n

Keep the vault inside a project repository when its notes describe that\nproject. A kb/ directory beside the code travels with every clone, branch, and\npull request, and wordcell context and wordcell history read the same\nrepository through --repo. Your usual Git workflow already syncs it, so you\nneed only Resolve a merge conflict from this\nguide.

\n

Give the vault its own repository when it holds memory that spans projects,\nsuch as preferences, people, and decisions that every agent session should\nread. The rest of this guide covers that case, with the vault at ~/kb.

\n

Create a private repository

\n

The Git host stores a copy of every note, so choose a host you would trust with\nthe vault’s contents. A private GitHub repository is accessible only to you,\nthe people you share it with, and, for an organization repository, certain\norganization members\n(GitHub documentation,\nchecked September 26, 2026). A bare repository on a server you control\n(git init --bare) works the same way over SSH.

\n

These steps use the GitHub CLI.

\n
    \n
  1. Create the vault if you do not have one: wordcell init ~/kb.
  2. \n
  3. Change to the vault: cd ~/kb.
  4. \n
  5. Start a repository: git init -b main.
  6. \n
  7. Stage every file: git add -A.
  8. \n
  9. Commit: git commit -m "Start vault".
  10. \n
  11. Create the private repository and add it as origin:\ngh repo create kb --private --source=. --remote=origin.
  12. \n
  13. Push and set the upstream branch: git push -u origin main.
  14. \n
\n

With another host, create an empty private repository in its interface, then\nrun git remote add origin <repository URL> in place of step 6.

\n

Pull before you work

\n

On each other machine, clone the repository once:

\n
git clone <repository URL> ~/kb\n
\n

Pull at the start of each session so that you and your agents read the latest\nnotes:

\n
git -C ~/kb pull --no-rebase\n
\n

Git carries only the files. Exact search, links, and wordcell check read the\nMarkdown directly, so they are current after a pull. Keyword, semantic, and\nhybrid search read the optional local QMD index, which stays on each machine.\nWhere you use those modes, run wordcell index --root ~/kb after a pull to\nupdate the index incrementally. wordcell graph rebuild writes\n.wordcell/oh.sqlite, which Git ignores, so rebuild it on each machine that\nuses it.

\n

A running wordcell mcp server reads edits\nfrom an editor or Git on its next call, so you do not need to restart it after\na pull.

\n

Commit and push on a schedule

\n

Save this script as ~/bin/sync-vault.sh. It commits every change in the vault,\nmerges the remote branch, and pushes. When the merge conflicts, it restores the\nstate from before the pull, keeps your local commit, and exits 1. A scheduled\nrun never leaves conflict markers in your notes.

\n
#!/bin/sh\n# Commit local changes, merge the remote branch, and push.\n# On a merge conflict, restore the pre-merge state and exit 1.\nset -eu\nvault="${1:?usage: sync-vault.sh <vault directory>}"\ncd "$vault"\n\ngit add -A\nif ! git diff --cached --quiet; then\n  git commit --quiet -m "Sync vault from $(hostname)"\nfi\n\nif ! git pull --quiet --no-rebase --no-edit; then\n  if git rev-parse -q --verify MERGE_HEAD >/dev/null; then\n    git merge --abort\n    echo "sync-vault: merge conflict in $vault; resolve it by hand" >&2\n  fi\n  exit 1\nfi\n\ngit push --quiet\n
\n

git add -A stages every file under the vault. Keep files you do not want on\nthe remote outside the vault, or list them in a .gitignore file. The script\nmerges instead of rebasing, so git merge --abort returns the vault to the\nexact state it had before the pull, and the next run tries again.

\n

The job runs without a terminal, so Git must reach the remote without asking\nfor a password or passphrase. For a GitHub remote over HTTPS,\ngh auth setup-git configures the GitHub CLI as Git’s credential helper. Run\nthe script once by hand to confirm that it exits 0:

\n
sh ~/bin/sync-vault.sh ~/kb\n
\n

Schedule it with cron

\n

Run crontab -e and add this line to sync every 30 minutes:

\n
*/30 * * * * /bin/sh "$HOME/bin/sync-vault.sh" "$HOME/kb" >>"$HOME/sync-vault.log" 2>&1\n
\n

Schedule it with launchd on macOS

\n

Save this agent as ~/Library/LaunchAgents/com.example.wordcell-sync.plist.\nReplace /Users/you with your home directory, because each path must be\nabsolute.

\n
<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n<plist version="1.0">\n<dict>\n  <key>Label</key>\n  <string>com.example.wordcell-sync</string>\n  <key>ProgramArguments</key>\n  <array>\n    <string>/bin/sh</string>\n    <string>/Users/you/bin/sync-vault.sh</string>\n    <string>/Users/you/kb</string>\n  </array>\n  <key>StartInterval</key>\n  <integer>1800</integer>\n  <key>StandardOutPath</key>\n  <string>/Users/you/Library/Logs/wordcell-sync.log</string>\n  <key>StandardErrorPath</key>\n  <string>/Users/you/Library/Logs/wordcell-sync.log</string>\n</dict>\n</plist>\n
\n

Load it:

\n
launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/com.example.wordcell-sync.plist\n
\n

To stop it, run the same command with bootout in place of bootstrap.\nlaunchd skips an interval that falls while the Mac is asleep and runs the job\nat the next one.

\n

Resolve a merge conflict

\n

Git merges changes to separate parts of a file and stops when both sides\nchanged the same or neighboring lines\n(git-merge documentation,\nchecked September 26, 2026). Edits to different notes never conflict. Three\nconflicts are common in a vault:

\n\n

When the log shows sync-vault: merge conflict, resolve it by hand:

\n
    \n
  1. Change to the vault: cd ~/kb.
  2. \n
  3. Merge the remote branch: git pull --no-rebase.
  4. \n
  5. List the conflicted files: git status --short. Each one starts with UU,\nor with AA when both machines created it. UD or DU means one machine\ndeleted a note that the other edited. Step 8 keeps the edited note unless\nyou remove it first with git rm.
  6. \n
  7. Edit each conflicted note to keep the text you want, and delete the\n<<<<<<<, =======, and >>>>>>> lines.
  8. \n
  9. If index.md conflicts, take the remote side: git checkout --theirs index.md.\nThe next step regenerates its catalog from the notes.
  10. \n
  11. Rebuild the catalog: wordcell refresh --root ..
  12. \n
  13. Check the vault: wordcell check --root .. Fix anything it reports.
  14. \n
  15. Stage the result: git add -A.
  16. \n
  17. Commit the merge: git commit --no-edit.
  18. \n
  19. Push: sh ~/bin/sync-vault.sh ~/kb.
  20. \n
\n

To give up on a merge and return to your local commit, run git merge --abort.

\n

If you import from Supermemory, run\nwordcell import supermemory on one machine and\nlet its notes sync like any other file. A later import on any machine reports a\nnote that was edited after the last import as a conflict and leaves it\nunchanged.

\n

Search several vaults instead of syncing them

\n

When each project keeps its own kb/, you do not need to combine them to search\nacross them. List the vaults in a portfolio registry and search the ones you\nselect:

\n
wordcell portfolio search "retry policy" \\\n  --registry ./kb-portfolio.json \\\n  --workspace ~/src \\\n  --shared\n
\n

Portfolio search reads the checkouts under the workspace as they are on disk,\nso pull each repository first. --shared selects vaults whose registry\nvisibility is public or organization. Select a private vault with\n--vault owner/id. See Define a v1 registry\nand Search selected vaults.

\n", diff --git a/site/app/readme.generated.ts b/site/app/readme.generated.ts index 3b3f6a28..81bc35c3 100644 --- a/site/app/readme.generated.ts +++ b/site/app/readme.generated.ts @@ -1,5 +1,5 @@ // Generated from ../README.md by scripts/sync-readme.ts. Do not edit. -export const readmeVersion = "0.26.3"; +export const readmeVersion = "0.27.0"; export const readmeTitle = "Wordcell"; export const readmeLead = "Wordcell is a Markdown knowledge base and local MCP memory server for coding agents such as Claude Code, Codex, and Cursor. It keeps decisions, plans, and sources as Markdown files beside your code, and agents find them by exact words, by meaning with an optional local model, or from the file they are about to change."; export const readmeHtml = "

Wordcell

\n
\n

🧠 Wordcell hands the next agent session what this one learned. Decisions,\nplans, and sources live as Markdown files beside your code. Tie a note to the\npaths it explains, and an agent about to change that code gets the note\nfirst. Search matches exact words or meaning, with an optional local model,\nand every index rebuilds from files you can read in any editor.

\n

Ask your agent to set it up: https://wordcell.io

\n
\n

Install the Agent Skill

\n

Wordcell is a Markdown knowledge base and local MCP memory server for coding\nagents such as Claude Code, Codex, and Cursor. It keeps decisions, plans, and\nsources as Markdown files beside your code, and agents find them by exact\nwords, by meaning with an optional local model, or from the file they are\nabout to change.

\n

A new coding-agent session needs the reasons behind the code it is changing.\nWordcell keeps the decision, its supporting sources, and the plan that puts it\ninto practice in a connected set of Markdown files. Tie a note to a repository\npath, and the next agent can recover that context before it edits the file.\nWhen an assumption changes, update the decision and keep its earlier reasoning\nin Git.

\n

Captured sources preserve what you read, maintained notes explain what you\ncurrently understand, and plans record the work and its results. Named\nrelationships such as evidenced-by and supersedes make those connections\navailable to the embedded Oh graph engine, whose results point back to the\nsupporting files. Optional local semantic search helps when the words in a\nquestion differ from the note; Git history connects the explanation to its\ncommits and accompanying code changes.

\n

Use the CLI, local MCP server, or TypeScript SDK from your existing agent and\neditor. The public Agent Skill teaches agents to recover context, maintain\nplans, and leave reusable decisions for later sessions. Exact search, backlinks,\nand Git history need no account or model. Every index rebuilds from files you\ncan read in Obsidian or any text editor. Wordcell is free and open source.

\n

Web capture, optional hosted reranking, and your agent's provider reach other\nservices; Privacy and boundaries says what each one\nsends.

\n

Documentation · Comparisons · Measured evidence · Changelog

\n

Why Wordcell

\n\n

Plain Markdown may be enough for a small set of notes. QMD is a good fit for\nlocal document retrieval and also supplies Wordcell's optional semantic search.\nWordcell adds a connected workflow for repository context, authored relationships,\nGit evidence, and selective publishing. Compare the tradeoffs.

\n

Wordcell keeps the record in Markdown files you own, rebuilds every index from those files, and lets the next session start from the notes for the file it is changing: the design every Hraness project shares. The thread through hraness follows that design across the projects, and the ALGAL vision states the bet behind it.

\n

Install

\n

Bun 1.3.14 or newer and Git are required.\nThe CLI and TypeScript SDK run with Bun. Install the versioned GitHub archive:

\n
bun add --global --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.26.0/hraness-wordcell-0.26.0.tgz\nwordcell --help\n
\n

Prefer npm? The same release is mirrored there:

\n
npm install --global --ignore-scripts @hraness/wordcell@0.26.0\nwordcell --help\n
\n

Keep Bun in PATH for either installation. If your shell cannot find wordcell,\nadd your package manager's global executable directory to PATH and reopen the\nterminal. Optional semantic search, browser capture, and PDF tools have\nadditional prerequisites.

\n

Global Bun and npm installations on macOS and Linux check for updates daily\nbefore a command starts. Updates require the GitHub CLI (gh) to verify the\nrelease and preserve --ignore-scripts. An active command, MCP server, or\npreview server prevents replacement until it exits. Both installed commands\nshare the same settings:

\n
wordcell update status\nwordcell update check\nwordcell update\nwordcell update disable\nwordcell update enable\n
\n

Use HRANESS_NO_UPDATE=1 wordcell … for an invocation without automatic update\nrequests, including offline work. Help, version output, and SDK imports never\ncheck for updates. Source, project, copied, and Windows installations use manual\nupgrades. Exact Bun registry versions stay pinned until update enable; npm\ndoes not reliably retain the original global version constraint, so use\nupdate disable when you need a fixed global version. A versioned GitHub archive\ntracks verified releases by default. See the update reference.

\n

Keep one decision available to the next session

\n

Run this from a directory where you want a new kb/ folder. It creates one\nMarkdown note and finds it without a search model or hosted search service:

\n
wordcell init kb\nwordcell note create notes/parser-contract \\\n  --title "Parser contract" --type concept --tag architecture \\\n  --body "Parser retries stop after three attempts." --root kb\nwordcell search "parser retries" --root kb --mode exact\n
\n

The result includes notes/parser-contract and the saved retry constraint.\nOpen kb/notes/parser-contract.md to see the ordinary Markdown file. Wordcell\nadds a stable document_id in its frontmatter; you can edit the prose normally.

\n

Already have Markdown or an Obsidian vault? In v0.22.0 or newer, search\nthe existing folder without initialization or an index.md file:

\n
wordcell search "a phrase from your notes" --root /path/to/your/vault --mode exact\n
\n

Replace the path and phrase with your own. You don't need to initialize,\nconvert, or move the existing files to search them.

\n

Use with a coding agent

\n

After trying the CLI, install the public Agent Skill into a compatible agent,\nsuch as Claude Code, Codex, Cursor, or GitHub Copilot:

\n
bunx skills add hraness/wordcell#v0.26.0 --skill wordcell\n
\n

Then ask:

\n
Use Wordcell to search for "parser retries" in ./kb using exact mode.\nRead the matching note and explain the saved constraint.\n
\n

The skill installs instructions, not a service. Installation does not create a\nvault, modify your notes, or grant an agent permission to access other accounts.\nThe agent still follows its own provider and data-handling settings.\nInspect the skill.

\n

Recover the stopped session

\n

Link a plan to the decision you saved:

\n
wordcell note create plans/parser-v2 \\\n  --title "Parser v2" --type plan \\\n  --body "The plan implements [[notes/parser-contract|the parser contract]]." \\\n  --root kb\nwordcell backlinks notes/parser-contract --root kb\n
\n

The backlink result includes plans/parser-v2, so a later session can find the\nwork that depends on the constraint.

\n

For code-path lookup, add this field inside the existing frontmatter of\nkb/notes/parser-contract.md:

\n
repository_scopes:\n  - packages/parser\n
\n

From your repository root, use an actual path under that scope and inspect the\nreturned notes and inherited AGENTS.md guides:

\n
wordcell context packages/parser/src/index.ts --root kb --repo .\n
\n

Commit the vault with your repository to preserve its history, then use\nwordcell history notes/parser-contract --root kb --repo . to inspect the\ncommits behind the note. History is optional and requires recorded Git commits.

\n

These views recover saved decisions, related work, rules, and provenance.\nThey do not reconstruct private chat or prove that the note is still correct.\nOpen the returned Markdown and guides before acting on them.

\n

Rerank a search window

\n

Cloudflare Clef support is in the current source tree. The versioned release\nlisted above does not include it. From a source checkout, run:

\n
bun ./src/cli.ts search "why releases use immutable archives" --root kb --mode exact \\\n  --rerank clef --rerank-limit 25 --limit 5 --json\n
\n

This optional hosted reranker uses Cloudflare Clef (clef by default, or\n--rerank-model clef-flash). It sends the query and candidate note identifiers,\ntitles, paths, and snippets to Cloudflare, needs CLOUDFLARE_ACCOUNT_ID and\nCLOUDFLARE_API_TOKEN, and incurs provider charges. Enable it only for vaults you\napprove for external processing; credentials alone never enable it. Exact\nidentities remain first; a provider failure retains the baseline order with a diagnostic. See the\nsetup, SDK examples, measured results, and limits.

\n

What you can do

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
TaskCommandEvidence and effects
Find a saved decisionwordcell search "parser retries" --root kb --mode exactReads current Markdown; no model or hosted search request.
Recover context for codewordcell context packages/parser/src/index.ts --root kb --repo .Returns scoped notes, plans, and inherited AGENTS.md rules.
Inspect explicit connectionswordcell backlinks notes/parser-contract --root kbReturns notes that link to the decision.
Search by meaningwordcell search "retry policy" --root kb --mode hybridAdds optional local QMD keyword and vector retrieval; model setup is required.
Query with graph proofswordcell graph query --program scope-route --scope packages/parser --root kbReturns bounded results tied to the source revision. Queries do not write a cache.
Capture a sourcewordcell clip https://example.com/article --output kb/articlesReads the selected URL and writes a Markdown bundle with a capture receipt.
Capture a PDFwordcell pdf /absolute/path/to/document.pdf --output kb/articlesPreserves the original PDF and extracted evidence; Poppler is required.
Check the vaultwordcell check --root kbReports structural and attachment problems without editing files.
Publish selected noteswordcell publish --root kb --out site/ --include notes/parser-contract --dry-run --jsonPreviews a static site selection locally; remove --dry-run to build it.
Preview a sitewordcell serve --root site --port 8080Serves a published site on a loopback static file server with the emitted 404.html fallback.
Connect an MCP clientwordcell mcp --root kbServes this vault to a local MCP client over standard input and output; --read-only removes the write tools.
Import from Supermemorywordcell import supermemory documents.json --root kbTurns saved Supermemory API responses into Markdown notes and reports local edits as conflicts on later imports.
\n

Use --json for structured output and wordcell --help for the complete command\nsurface. Full command reference.

\n

Publish a selected part of your vault

\n

Preview the decision and plan from the example before writing an output folder:

\n
wordcell publish --root kb --out site \\\n  --include notes/parser-contract --include plans/parser-v2 --dry-run --json\n
\n

Review the selection, then build and preview it:

\n
wordcell publish --root kb --out site \\\n  --include notes/parser-contract --include plans/parser-v2\nwordcell serve --root site --port 8080\n
\n

Open http://127.0.0.1:8080. You get readable pages, linked notes, and search\nthat runs in the browser. Upload the site/ folder to your chosen static host\nwhen you want to share it; publish itself never uploads anything.

\n

For a repeatable slice, use path patterns or combine folders, tags, metadata,\ncode scopes, and linked neighborhoods:

\n
wordcell publish --root kb --out site-notes \\\n  --include-glob 'notes/**/*.md' --exclude-glob '**/draft-*' --dry-run --json\n
\n

Preview reports show up to 20 selected IDs by default, with a total count and\nselection digest. They keep note bodies out of the agent's context. Use\n--list-limit to adjust that preview without changing what gets published. publish: false excludes a note, but selected\nprose and attachments still need review before sharing: selection is not secret\nredaction. Selection recipes and hosting guide.

\n

Use a vault as agent memory

\n

wordcell mcp --root kb serves a vault to a local MCP client, such as Claude\nCode, Claude Desktop, Cursor, or Codex, over standard input and output. The\nclient can search, list, and read notes, follow links, create notes, update a\nnote body at a known revision, and add typed relations. --read-only removes\nthe write tools. Connect Wordcell to your coding agent\nshows the setup for each host.

\n

With the Agent Skill, ask your agent to save the session. It writes a dated\nsession note, links it to the notes it changed, and keeps a profile note for\nyou or your project. Wordcell extracts no facts and writes no note on its own.\nSession memory and profiles.

\n

To move from Supermemory, wordcell import supermemory turns saved Supermemory\nAPI responses into Markdown notes. The migration guide\ncovers export, import, and what does not transfer, and the\nSupermemory comparison lists when to\nchoose Supermemory instead. Sync with Git\nkeeps one vault current on several machines.

\n

Evidence and comparisons

\n

In the September 19, 2026 SciFact study of scientific abstracts, hosted Jev\nreranking placed a relevant result first for 161 of 300 queries, versus\n101 with Wordcell exact search alone. These historical measurements do not\nevaluate Cloudflare Clef or Clef Flash. Results and method.

\n

The comparison guide\ncovers Markdown alone, QMD, Basic Memory, Obsidian, static publishing tools,\nSupermemory, Mem0, and Zep using their own documentation. Choose the smallest\nworkflow that meets your needs.

\n

The benchmarks page shows the SciFact results\nand the published results of Oh, the embedded memory framework, each with its\nsource data and method. On all 500 LongMemEval-S\nquestions, Oh semantic retrieval scored 88.87% and BM25 86.13% with the same\nreader and budget; on the measure Oh named before the run, its interval does\nnot rule out a tie. Oh's scores measure its own memory-retrieval path, not a\nWordcell vault.

\n

How the files fit together

\n
repository/\n├── AGENTS.md                # rules that govern edits\n├── packages/parser/\n│   └── AGENTS.md            # rules scoped to this code\n└── kb/\n    ├── index.md             # authored or managed front door\n    ├── articles/            # captured sources and assets\n    ├── notes/               # maintained explanations\n    └── plans/               # decisions and outcomes\n
\n

Markdown, YAML frontmatter, explicit wikilinks, and Git hold the record. Open\nthe same files in Obsidian, a text editor, or ordinary file-search tools.\nApplication code does not need to import Wordcell or its vault.

\n

Wordcell is the Markdown knowledge base. Oh is the\nembedded memory framework that backs its named graph queries and source proofs.\nMarkdown and Git remain authoritative. Query the graph immediately without an\nOh account, service, or persisted database:

\n
wordcell graph query --program backlinks --note notes/parser-contract --root kb --json\n
\n

The result traces each returned link to its source note and revision. Only an explicit\nwordcell graph rebuild --root kb writes .wordcell/oh.sqlite, the file stays\nignored and rebuildable, and nothing flows from the projection back into notes.\nBacklinks and typed relationships come from authored links. Percolation\nsuggests connections for review and does not add inferred edges to notes.

\n

Wordcell search combines its own exact matching with optional QMD local search\nand optional hosted Cloudflare Clef reranking. Oh also offers memory retrieval for applications;\nits conversation-memory benchmark scores measure that separate path. They do not\nestablish Wordcell's retrieval or answer quality. How the integration works.

\n

Graph proofs explain a supported derivation from a specific source revision.\nThey do not prove that a note is true or that a missing relationship cannot\nexist. Graph queries and proof limits.

\n

Build with the TypeScript SDK

\n

Add the same immutable release to a Bun project:

\n
bun add --exact --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.26.0/hraness-wordcell-0.26.0.tgz\n
\n

The SDK provides read-only vault sessions, metadata queries, search, graph\nproofs, Git context, and composable workflows. A session owns one snapshot;\nreopen it after Markdown changes. SDK and workflow examples\nshow the public imports and lifecycle.

\n

Privacy and boundaries

\n\n

Documentation

\n

Start with the task you want to complete. Browse all guides on the\ndocumentation index.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Read nextPurpose
Get startedLearn the full loop on a first vault: save, find, connect, and publish one note.
Agent workflowSet up, query, maintain, and revise repository memory.
Search an existing Obsidian or Markdown vaultFind a saved note without initialization or a model download.
Use an existing Obsidian vaultSee which Obsidian links, tags, and attachments Wordcell reads, and serve the vault to an agent read-only.
Connect a coding agent over MCPConfigure Claude Code, Codex, Cursor, or Claude Desktop, then verify a note read.
Troubleshoot MCPDiagnose executable paths, partial search results, missing tools, and oversized responses.
Installation and command referenceExact interfaces, SDK imports, optional adapters, and troubleshooting prerequisites.
Web capture and PDF captureSave sources with provenance, assets, and explicit completeness limits.
Publish selected notesPreview a slice, build a static site, and choose how to host it.
Graph guideNamed queries, proofs, revisions, resource limits, and cache recovery.
Search several vaults togetherSearch only selected, authorized vaults.
Sync with GitKeep one vault current on several machines with a private repository.
Migrate from SupermemoryExport documents and memory entries, import them as notes, and replace connectors.
Design and memory rationaleFile contracts, design choices, and evaluation context.
Release verificationVerify archive identity, signatures, and provenance.
\n

Installation reference

\n

Alternative installs and optional capabilities.

\n

The kb vault format

\n

Existing kb/, index.md, kb:// identifiers, and Markdown formats remain\ncompatible. Vault format reference.

\n

Command surface

\n

Complete CLI reference.

\n

Capture reference

\n

Capture commands and dependencies.

\n

Graph reference

\n

Graph and SDK interfaces.

\n

Agent skills

\n

Installation and runtime contract.\nThe same skill is included at node_modules/@hraness/wordcell/skills/wordcell/.

\n

Release notes

\n

Version 0.22 adds selective static publishing and local preview. Choose notes,\nfolders, path patterns, metadata, or linked neighborhoods, then inspect a\nbounded selection report before building. Existing vaults need no migration.\nThe deprecated kb command was removed in 0.21.0; use wordcell.\nAll release notes and upgrade instructions.

\n

Contributing

\n

Run bun run check for source, architecture, package installation, types, and\nnative-helper validation. The site has its own bun run check under site/.\nSee Contributing, Security, and the MIT License.

\n

Install Wordcell from GitHub Releases ·\nWordcell source on GitHub ·\nWordcell overview

\n

Optional development support

\n

wordcell support shows optional development support for Wordcell. Features stay\nfree and no account is required. Accounts presents current terms; the person\nconfirms any payment in their browser. Wordcell has no product newsletter and\ndoes not look up an email address for support.

\n

After useful standalone work, agents can receive a discovery record on stderr.\nRead wordcell support protocol --json for the shared closeout procedure.\nHelp, diagnostics, evaluation, SDK calls and nested commands stay quiet.\nHRANESS_SUPPORT=off or HRANESS_SUPPORT_AUDIENCE=off suppresses incidental\ninvitations. Explicit human terminal rendering requires\nHRANESS_SUPPORT_AUDIENCE=human; the default audience is an agent, including in\na pseudo-terminal.

\n

wordcell support dismiss disables invitations across participating tools on\nthis machine. snooze pauses them for thirty days, enable restores them, and\nstatus --json shows their separate local preferences. These commands do not\nchange a vault, sign up or pay. Acknowledged invitations share a seven-day\ncooldown; discovery itself does not consume it.

\n"; diff --git a/skills/wordcell/SKILL.md b/skills/wordcell/SKILL.md index a9691834..0ef98a6b 100644 --- a/skills/wordcell/SKILL.md +++ b/skills/wordcell/SKILL.md @@ -58,7 +58,7 @@ missing: ```sh command -v wordcell >/dev/null 2>&1 || { command -v bun >/dev/null 2>&1 || exit 1 - bun add --global --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.26.3/hraness-wordcell-0.26.3.tgz + bun add --global --ignore-scripts https://github.com/hraness/wordcell/releases/download/v0.27.0/hraness-wordcell-0.27.0.tgz } wordcell --help ``` diff --git a/src/capture.ts b/src/capture.ts index 34184ae4..8d89b5ad 100644 --- a/src/capture.ts +++ b/src/capture.ts @@ -66,3 +66,8 @@ export type { MediaMode, ParseArgumentsResult, }; + +import { extractArticleMetadata as extractArticleMetadataImplementation, parseArticleMetadata as parseArticleMetadataImplementation } from "./clip/metadata.js"; +export const extractArticleMetadata: typeof extractArticleMetadataImplementation = (...arguments_) => extractArticleMetadataImplementation(...arguments_); +export const parseArticleMetadata: typeof parseArticleMetadataImplementation = (...arguments_) => parseArticleMetadataImplementation(...arguments_); +export type { ArticleMetadata, ArticleMetadataClaim, MetadataSource, MetadataValue } from "./clip/metadata.js"; diff --git a/src/clip/bundle-reader.ts b/src/clip/bundle-reader.ts index d39d3ada..a0f99129 100644 --- a/src/clip/bundle-reader.ts +++ b/src/clip/bundle-reader.ts @@ -1,3 +1,4 @@ +import { parseArticleMetadata, type ArticleMetadata } from "./metadata.js"; import { createHash } from "node:crypto"; import { constants as fsConstants, type BigIntStats } from "node:fs"; import { lstat, open, opendir, realpath, type FileHandle } from "node:fs/promises"; @@ -27,6 +28,7 @@ const SHA256_PATTERN = /^[0-9a-f]{64}$/u; export type CaptureBundleIntegrity = "mismatch" | "unavailable" | "verified"; export type CaptureBundleInspection = { + readonly metadata?: ArticleMetadata; readonly root: string; readonly schemaVersion: 1 | 2 | 3 | 4; readonly sourceUrl: string; @@ -556,6 +558,7 @@ export async function readCaptureBundle( await assertBundleRoot(root); return Object.freeze({ + ...(manifest.metadata === undefined ? {} : { metadata: parseArticleMetadata(manifest.metadata) }), root: root.path, schemaVersion, sourceUrl: manifest.sourceUrl, diff --git a/src/clip/capture.ts b/src/clip/capture.ts index e618330e..0814949a 100644 --- a/src/clip/capture.ts +++ b/src/clip/capture.ts @@ -965,6 +965,7 @@ export async function runCapture( const finalWarnings = finalizedWarnings(combinedWarnings, redactedMarkdown.count); const includeSource = options.evidence === "source" || options.evidence === "all"; const manifestInput: CaptureManifestInput = { + ...(article.metadata === undefined ? {} : { metadata: article.metadata }), sourceUrl, canonicalUrl: best.canonicalUrl.href, capturedAt, diff --git a/src/clip/defuddle-worker.ts b/src/clip/defuddle-worker.ts index 38a64d7e..7a9849ba 100644 --- a/src/clip/defuddle-worker.ts +++ b/src/clip/defuddle-worker.ts @@ -1,3 +1,4 @@ +import { extractArticleMetadata } from "./metadata.js"; import { Defuddle } from "defuddle/node"; import { embeddedArticleHtml } from "./embedded-article.js"; @@ -125,7 +126,7 @@ workerGlobal.onmessage = async (event: MessageEvent): Promise => removeImages: false, }); const enriched = typeof value === "object" && value !== null && !Array.isArray(value) - ? { ...value, captureVideoPosters: videoPosters } + ? { ...value, captureVideoPosters: videoPosters, metadata: await extractArticleMetadata(request.html, request.url, value) } : value; workerGlobal.postMessage({ ok: true, value: enriched }); } catch { diff --git a/src/clip/extract.ts b/src/clip/extract.ts index 19166ca1..3f91ba5e 100644 --- a/src/clip/extract.ts +++ b/src/clip/extract.ts @@ -1,3 +1,4 @@ +import { parseArticleMetadata } from "./metadata.js"; import type { CaptureScope } from "./args.js"; import type { AcquiredPage } from "./acquire.js"; import { @@ -719,6 +720,7 @@ export async function extractPage( content: retainDefuddleMedia(restoredContent, response, acquisition.finalUrl), title: boundedMetadata(response.title, articleMetadataLimits.title) ?? boundedMetadata(acquisition.browserTitle, articleMetadataLimits.title), + ...(response.metadata === undefined ? {} : { metadata: parseArticleMetadata(response.metadata) }), author: boundedMetadata(response.author, articleMetadataLimits.author), published: boundedMetadata(response.published, articleMetadataLimits.published), description, diff --git a/src/clip/lib.ts b/src/clip/lib.ts index 72d6da84..349f8d6c 100644 --- a/src/clip/lib.ts +++ b/src/clip/lib.ts @@ -1,7 +1,9 @@ +import { parseArticleMetadata, type ArticleMetadata } from "./metadata.js"; import { sanitizeTerminalText } from "./terminal.js"; /** Defuddle fields consumed after narrowing its JSON output. */ export type Article = { + metadata?: ArticleMetadata; content: string; title: string | null; author: string | null; @@ -46,6 +48,7 @@ export function parseArticle(parsed: unknown): Article | null { if (content === null) return null; return { content, + ...(parsed.metadata === undefined ? {} : { metadata: parseArticleMetadata(parsed.metadata) }), title: nonEmptyString(parsed.title, articleMetadataLimits.title), author: nonEmptyString(parsed.author, articleMetadataLimits.author), published: nonEmptyString(parsed.published, articleMetadataLimits.published), diff --git a/src/clip/metadata-search.test.ts b/src/clip/metadata-search.test.ts index adee0513..cd722ee6 100644 --- a/src/clip/metadata-search.test.ts +++ b/src/clip/metadata-search.test.ts @@ -322,7 +322,7 @@ process.stdout.write`, const provider = createRustMetadataSearchProvider({ binaryPath: fixture.path, resolveNetworkTarget: async () => shuffled }); expect((await provider({ query: "large DNS", timeoutMs: 10_000 })).status).toBe("success"); }), { numRuns: 8 }); - }); + }, 30_000); test("rejects unsafe DNS answers beyond the candidate limit before spawning", async () => { const fixture = executable(SUCCESS_SCRIPT); diff --git a/src/clip/metadata.test.ts b/src/clip/metadata.test.ts new file mode 100644 index 00000000..e093fc24 --- /dev/null +++ b/src/clip/metadata.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, test } from "bun:test"; +import { extractArticleMetadata, parseArticleMetadata, metadataLimits } from "./metadata.js"; + +describe("observed article metadata", () => { + test("retains conflicts, authors and evidence with deterministic choice", async () => { + const metadata = await extractArticleMetadata(`HTML & title + + + + + `, "https://example.test/post"); + const selected = metadata.claims.filter((claim) => claim.selected); + expect(selected.find((claim) => claim.field === "title")?.value).toBe("LD title"); + expect(metadata.claims.filter((claim) => claim.field === "title")).toHaveLength(4); + expect(selected.find((claim) => claim.field === "authors")?.value).toEqual(["Ada", "Bo"]); + expect(selected.find((claim) => claim.field === "published")).toMatchObject({ value: "2026-10-16T04:30:00.000Z", timezone: "explicit", datePrecision: "datetime", source: "json-ld" }); + expect(selected.find((claim) => claim.field === "description")?.value).toBe("Description é"); + expect(selected.find((claim) => claim.field === "canonical")?.value).toBe("https://example.test/article"); + expect(metadata.claims.every((claim) => claim.locator && claim.rule)).toBe(true); + }); + test("retains timezone uncertainty and never invents publication time", async () => { + for (const [date, precision, timezone] of [["2026-10-16", "date", "not-applicable"], ["2026-10-16T12:30:00", "datetime", "unknown"], ["2026-02-31", "invalid", "unknown"]]) { + const metadata = await extractArticleMetadata(``, "https://example.test/"); + expect(metadata.claims[0]).toMatchObject({ value: date, datePrecision: precision, timezone, selected: precision !== "invalid" }); + } + expect((await extractArticleMetadata("No date", "https://example.test/")).claims.some((claim) => claim.field === "published")).toBe(false); + }); + test("bounded JSON-LD ignores remote contexts, hostile depth and malformed scripts", async () => { + const deep = '{"x":'.repeat(20) + '0' + '}'.repeat(20); + const metadata = await extractArticleMetadata(``, "https://example.test/"); + expect(metadata.warnings).toContain("json-limit"); expect(metadata.warnings).toContain("json-invalid"); + expect(JSON.stringify(metadata)).not.toContain("secret"); expect(JSON.stringify(metadata)).not.toContain("pass@"); + expect(metadata.claims).toHaveLength(1); + }); + test("enforces claim/value limits and rejects unknown envelopes", async () => { + const metadata = await extractArticleMetadata(''.repeat(metadataLimits.claims + 20), "https://example.test/"); + expect(metadata.claims.length).toBeLessThanOrEqual(metadataLimits.claims); expect(metadata.warnings).toContain("claim-limit"); + expect(() => parseArticleMetadata({ ...metadata, schemaVersion: 2 })).toThrow(); + expect(() => parseArticleMetadata({ ...metadata, claims: [{ ...metadata.claims[0], value: "x".repeat(9000) }] })).toThrow(); + expect(() => parseArticleMetadata({ ...metadata, claims: [{ ...metadata.claims[0], selected: "yes" }] })).toThrow(); + }); +}); + + +test("unknown meta keys do not inherit object prototype fields", async () => { + const metadata = await extractArticleMetadata('Still captured', "https://example.test/"); + expect(metadata.claims.find(claim => claim.field === "meta:constructor")?.value).toBe("A builder"); + expect(metadata.claims.find(claim => claim.field === "title")?.value).toBe("Still captured"); +}); +test("an unterminated JSON-LD script is explicitly incomplete", async () => { + const metadata = await extractArticleMetadata('Useful title`, "https://example.test/"); + expect(metadata.claims.find(claim => claim.field === "title" && claim.selected)?.value).toBe("Actual article"); + expect(metadata.claims.find(claim => claim.field === "description" && claim.selected)?.value).toBe("Article details"); + expect(metadata.claims.find(claim => claim.observed === "Generic page")?.rule).toContain("jsonld-type=WebPage"); + expect(metadata.claims.filter(claim => claim.field === "title")).toHaveLength(2); + } +}); diff --git a/src/clip/metadata.ts b/src/clip/metadata.ts new file mode 100644 index 00000000..87072fd9 --- /dev/null +++ b/src/clip/metadata.ts @@ -0,0 +1,232 @@ +import { decodeHTML } from "entities"; + +export const metadataLimits = { htmlBytes: 8 * 1024 * 1024, claims: 256, value: 8192, totalBytes: 256 * 1024, jsonBytes: 128 * 1024, jsonNodes: 2048, jsonDepth: 12 } as const; +export type MetadataSource = "html-title" | "html-meta" | "html-link" | "html-language" | "json-ld" | "extractor"; +export type MetadataValue = string | readonly string[]; +export type ArticleMetadataClaim = { + readonly field: string; + readonly observed: MetadataValue; + readonly value: MetadataValue; + readonly source: MetadataSource; + readonly locator: string; + readonly rule: string; + readonly selected: boolean; + readonly datePrecision?: "date" | "datetime" | "invalid"; + readonly timezone?: "explicit" | "unknown" | "not-applicable"; +}; +export type ArticleMetadata = { + readonly schemaVersion: 1; + readonly extractorVersion: "wordcell-metadata-v1"; + readonly claims: readonly ArticleMetadataClaim[]; + readonly warnings: readonly ("input-limit" | "claim-limit" | "value-limit" | "json-limit" | "json-invalid")[]; +}; +const record = (v: unknown): v is Record => typeof v === "object" && v !== null && !Array.isArray(v); +const sources = new Set(["html-title", "html-meta", "html-link", "html-language", "json-ld", "extractor"]); +const warningKinds = new Set(["input-limit", "claim-limit", "value-limit", "json-limit", "json-invalid"]); +const credentialName = /(?:csrf|xsrf|token|nonce|password|secret|session|authorization|api[-_]?key|signature)/i; +const urlFields = new Set(["canonical", "alternate", "image", "url"]); +const dateFields = new Set(["published", "modified"]); + +/** Metadata is descriptive evidence, never an authenticated resource locator. */ +function publicUrl(value: string, base?: string): string | null { + try { + const url = new URL(value, base); + if (!/^https?:$/.test(url.protocol)) return null; + url.username = ""; url.password = ""; url.search = ""; url.hash = ""; + return url.href; + } catch { return null; } +} +function cleanText(value: string): string { + return value.replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f-\u009f]/gu, "") + .replace(/https?:\/\/[^\s<>"']+/giu, (url) => publicUrl(url) ?? "[invalid URL]") + .replace(/\s+/gu, " ").trim(); +} +function dateValue(value: string): Pick { + const dateOnly = /^\d{4}-\d{2}-\d{2}$/.test(value); + const zoned = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d{1,9})?)?(?:Z|[+-]\d{2}:\d{2})$/i.test(value); + const local = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d{1,9})?)?$/.test(value); + const parsed = Date.parse(dateOnly ? value + "T00:00:00Z" : local ? value + "Z" : value); + const validDay = Number.isFinite(parsed) && new Date(Date.UTC(Number(value.slice(0, 4)), Number(value.slice(5, 7)) - 1, Number(value.slice(8, 10)))).toISOString().slice(0, 10) === value.slice(0, 10); + if ((!dateOnly && !zoned && !local) || !validDay) return { value, datePrecision: "invalid", timezone: "unknown" }; + if (dateOnly) return { value, datePrecision: "date", timezone: "not-applicable" }; + return { value: zoned ? new Date(parsed).toISOString() : value, datePrecision: "datetime", timezone: zoned ? "explicit" : "unknown" }; +} +function rank(claim: ArticleMetadataClaim): number { + if (claim.datePrecision === "invalid") return 1000; + if (claim.source === "json-ld") return claim.rule.includes("jsonld-type=WebPage") ? 15 : 10; + if (claim.source === "html-link") return 5; + if (claim.locator.includes("property=article:")) return 20; + if (claim.locator.includes("property=og:")) return 30; + if (claim.locator.includes("name=twitter:")) return 40; + if (claim.source === "html-meta") return 50; + if (claim.source === "html-title" || claim.source === "html-language") return 60; + return 90; +} +function select(claims: readonly ArticleMetadataClaim[]): ArticleMetadataClaim[] { + const ranks = new Map(); + for (const claim of claims) ranks.set(claim.field, Math.min(ranks.get(claim.field) ?? 1000, rank(claim))); + const chosen = new Set(); + return claims.map((claim) => { + const multiple = ["authors", "tags", "alternate"].includes(claim.field); + const selected = rank(claim) < 1000 && rank(claim) === ranks.get(claim.field) && (multiple || !chosen.has(claim.field)); + if (selected) chosen.add(claim.field); + return { ...claim, selected, rule: `metadata-precedence-v1:${claim.rule}` }; + }); +} + +/** Validate an optional additive envelope from a worker or a stored bundle. */ +export function parseArticleMetadata(input: unknown, redact: (text: string) => string = cleanText): ArticleMetadata { + if (!record(input) || input.schemaVersion !== 1 || input.extractorVersion !== "wordcell-metadata-v1" || !Array.isArray(input.claims) || input.claims.length > metadataLimits.claims || !Array.isArray(input.warnings)) throw new Error("Unsupported or malformed article metadata envelope"); + let bytes = 0; + const text = (value: unknown, limit: number = metadataLimits.value): string => { + if (typeof value !== "string" || value.length > limit) throw new Error("Invalid article metadata text"); + const result = cleanText(redact(value)); + bytes += Buffer.byteLength(result); + if (bytes > metadataLimits.totalBytes) throw new Error("Article metadata exceeds byte limit"); + return result; + }; + const value = (v: unknown): MetadataValue => Array.isArray(v) + ? v.length <= 32 ? v.map((part) => text(part)) : (() => { throw new Error("Article metadata list exceeds limit"); })() + : text(v); + const claims = input.claims.map((raw): ArticleMetadataClaim => { + if (!record(raw) || !sources.has(raw.source as MetadataSource) || typeof raw.selected !== "boolean") throw new Error("Invalid article metadata claim"); + const field = text(raw.field, 128); + if (!/^[a-z][a-z0-9:_-]{0,127}$/.test(field) || credentialName.test(field)) throw new Error("Invalid article metadata field"); + let observed = value(raw.observed), normalized = value(raw.value); + if (urlFields.has(field)) { + const urls = (input: MetadataValue): MetadataValue => { + const normalize = (part: string): string => { + const parsed = publicUrl(part); + if (parsed === null) throw new Error("Invalid article metadata URL"); + return parsed; + }; + return typeof input === "string" ? normalize(input) : input.map(normalize); + }; + observed = urls(observed); normalized = urls(normalized); + } + const temporal = raw.datePrecision === undefined ? {} : dateFields.has(field) && typeof observed === "string" + ? dateValue(observed) : (() => { throw new Error("Invalid article metadata date"); })(); + if (raw.datePrecision !== undefined && (raw.datePrecision !== (temporal as ArticleMetadataClaim).datePrecision || raw.timezone !== (temporal as ArticleMetadataClaim).timezone)) throw new Error("Invalid article metadata date precision"); + return { field, observed, value: normalized, source: raw.source as MetadataSource, locator: text(raw.locator, 512), rule: text(raw.rule, 256), selected: raw.selected, ...temporal }; + }); + if (input.warnings.length > warningKinds.size || input.warnings.some((warning) => !warningKinds.has(warning))) throw new Error("Invalid article metadata warnings"); + return { schemaVersion: 1, extractorVersion: "wordcell-metadata-v1", claims, warnings: [...input.warnings] as ArticleMetadata["warnings"] }; +} + +const fields: Readonly> = { + "og:title": "title", "twitter:title": "title", title: "title", "citation_title": "title", "dc.title": "title", + description: "description", "og:description": "description", "twitter:description": "description", "dc.description": "description", + author: "authors", "article:author": "authors", "citation_author": "authors", "dc.creator": "authors", + "article:published_time": "published", "datepublished": "published", "citation_publication_date": "published", "dc.date": "published", + "article:modified_time": "modified", "datemodified": "modified", "last-modified": "modified", + "og:url": "canonical", "twitter:url": "canonical", "og:image": "image", "og:image:url": "image", "twitter:image": "image", "twitter:image:src": "image", + "og:site_name": "site", publisher: "publisher", "dc.publisher": "publisher", "citation_journal_title": "publisher", + "og:locale": "language", language: "language", "dc.language": "language", "og:type": "type", "twitter:card": "card", + "article:tag": "tags", keywords: "tags", "news_keywords": "tags", "citation_keywords": "tags", + "citation_doi": "doi", "dc.identifier": "identifier", "citation_pdf_url": "alternate", +}; + +/** Parse inert HTML locally. JSON-LD contexts are evidence strings and are never resolved. */ +export async function extractArticleMetadata(html: string, base: string, fallback: { readonly title?: unknown; readonly author?: unknown; readonly published?: unknown; readonly description?: unknown } = {}): Promise { + const claims: ArticleMetadataClaim[] = []; + const warnings = new Set(); + let totalBytes = 0, title = "", script = "", scriptIndex = 0, activeScript = false, jsonBytes = 0; + const add = (field: string, raw: unknown, source: MetadataSource, locator: string, decode = false, evidenceRule?: string): void => { + if (claims.length >= metadataLimits.claims) { warnings.add("claim-limit"); return; } + let values = Array.isArray(raw) ? raw : [raw]; + if (values.length > 32) { warnings.add("value-limit"); values = values.slice(0, 32); } + const strings: string[] = []; + for (const value of values) { + if (typeof value !== "string") continue; + if (value.length > metadataLimits.value) { warnings.add("value-limit"); continue; } + const decoded = decode ? decodeHTML(value) : value; + const cleaned = cleanText(urlFields.has(field) ? publicUrl(decoded, base) ?? "" : decoded); + if (cleaned) strings.push(cleaned); + } + if (!strings.length) return; + const observed: MetadataValue = Array.isArray(raw) ? strings : strings[0]!; + let value: MetadataValue = observed; + if (urlFields.has(field)) { + const urls = strings.map((part) => publicUrl(part, base)).filter((part): part is string => part !== null); + if (!urls.length) return; + value = Array.isArray(raw) ? urls : urls[0]!; + } + const temporal = dateFields.has(field) && typeof observed === "string" ? dateValue(observed) : {}; + const claim: ArticleMetadataClaim = { field, observed, value, source, locator, rule: `trim-whitespace-url-query-redaction${evidenceRule === undefined ? "" : `:${evidenceRule}`}`, selected: false, ...temporal }; + const bytes = Buffer.byteLength(JSON.stringify(claim)); + if (totalBytes + bytes > metadataLimits.totalBytes / 2) { warnings.add("claim-limit"); return; } + totalBytes += bytes; claims.push(claim); + }; + const parseLd = (raw: string, index: number): void => { + if (!raw.trim()) return; + if (Buffer.byteLength(raw) > metadataLimits.jsonBytes || (jsonBytes += Buffer.byteLength(raw)) > metadataLimits.jsonBytes * 4) { warnings.add("json-limit"); return; } + let parsed: unknown; + try { parsed = JSON.parse(raw); } catch { warnings.add("json-invalid"); return; } + let nodes = 0; + // A full bounded walk rejects deep or wide hostile payloads before extracting facts. + const stack: { value: unknown; depth: number }[] = [{ value: parsed, depth: 0 }]; + while (stack.length) { + const node = stack.pop()!; + if (++nodes > metadataLimits.jsonNodes || node.depth > metadataLimits.jsonDepth) { warnings.add("json-limit"); return; } + const children = Array.isArray(node.value) ? node.value : record(node.value) ? Object.values(node.value) : []; + if (children.length + nodes + stack.length > metadataLimits.jsonNodes) { warnings.add("json-limit"); return; } + for (const child of children) stack.push({ value: child, depth: node.depth + 1 }); + } + const items: { value: unknown; path: string }[] = Array.isArray(parsed) ? parsed.map((value, i) => ({ value, path: `$[${i}]` })) : [{ value: parsed, path: "$" }]; + for (let cursor = 0; cursor < items.length && cursor < metadataLimits.jsonNodes; cursor++) { + const item = items[cursor]!; + if (!record(item.value)) continue; + const object = item.value; + if (Array.isArray(object["@graph"])) items.push(...object["@graph"].map((value, i) => ({ value, path: `${item.path}.@graph[${i}]` }))); + const types = Array.isArray(object["@type"]) ? object["@type"] : [object["@type"]]; + const entityTypes = types.flatMap((type) => { + const match = typeof type === "string" ? /^(?:(?:https?:\/\/)?schema\.org\/)?(Article|NewsArticle|BlogPosting|ScholarlyArticle|TechArticle|Report|WebPage|VideoObject|AudioObject|PodcastEpisode)$/.exec(type) : null; + return match?.[1] === undefined ? [] : [match[1]]; + }); + const entityType = entityTypes.find(type => type !== "WebPage") ?? entityTypes[0]; + if (entityType === undefined) continue; + const names = (v: unknown): unknown => Array.isArray(v) ? v.slice(0, 32).map((entry) => record(entry) ? entry.name : entry) : record(v) ? v.name : v; + const props: Readonly> = { headline: "title", name: "title", description: "description", author: "authors", creator: "authors", datePublished: "published", dateModified: "modified", inLanguage: "language", publisher: "publisher", keywords: "tags", image: "image", thumbnailUrl: "image", url: "canonical", mainEntityOfPage: "canonical", duration: "duration", episodeNumber: "episode", identifier: "identifier" }; + for (const [key, field] of Object.entries(props)) { + let value = object[key]; + if (value === undefined) continue; + if (["author", "creator", "publisher"].includes(key)) value = names(value); + if (key === "image") value = Array.isArray(value) ? value.map((v) => record(v) ? v.url ?? v.contentUrl : v) : record(value) ? value.url ?? value.contentUrl : value; + if (key === "mainEntityOfPage" && record(value)) value = value["@id"] ?? value.url; + if (key === "episodeNumber" && typeof value === "number" && Number.isFinite(value)) value = String(value); + add(field, value, "json-ld", `script[${index}]${item.path}.${key}`, false, `jsonld-type=${entityType}`); + } + } + }; + let input = html; + if (Buffer.byteLength(input) > metadataLimits.htmlBytes) { input = Buffer.from(input).subarray(0, metadataLimits.htmlBytes).toString("utf8"); warnings.add("input-limit"); } + await new HTMLRewriter() + .on("html", { element(element) { add("language", element.getAttribute("lang"), "html-language", "html@lang", true); } }) + .on("title", { text(chunk) { if (title.length <= metadataLimits.value) title += chunk.text; } }) + .on("meta", { element(element) { + const attribute = element.getAttribute("property") ? "property" : "name"; + const key = decodeHTML(element.getAttribute(attribute) ?? "").toLowerCase(); + if (!/^[a-z][a-z0-9:._-]{0,100}$/.test(key) || credentialName.test(key)) return; + const field = Object.hasOwn(fields, key) ? fields[key]! : `meta:${key.replaceAll(".", "_")}`; + add(field, element.getAttribute("content"), "html-meta", `meta[${attribute}=${key}]@content`, true); + } }) + .on("link", { element(element) { + const rel = (element.getAttribute("rel") ?? "").toLowerCase().split(/\s+/); + const field = rel.includes("canonical") ? "canonical" : rel.includes("alternate") ? "alternate" : null; + if (field) add(field, element.getAttribute("href"), "html-link", `link[rel=${field}]@href`, true); + } }) + .on("script", { element(element) { + activeScript = (element.getAttribute("type") ?? "").toLowerCase().trim() === "application/ld+json"; + script = ""; + if (activeScript) { + const index = scriptIndex++; + if (index >= 64) { activeScript = false; warnings.add("json-limit"); return; } + element.onEndTag(() => { parseLd(script, index); activeScript = false; }); + } + }, text(chunk) { if (activeScript && script.length <= metadataLimits.jsonBytes) script += chunk.text; } }) + .transform(new Response(input)).text(); + if (activeScript) warnings.add("json-invalid"); + add("title", title, "html-title", "title", true); + for (const [key, field] of Object.entries({ title: "title", author: "authors", published: "published", description: "description" })) add(field, fallback[key as keyof typeof fallback], "extractor", `defuddle.${key}`); + return parseArticleMetadata({ schemaVersion: 1, extractorVersion: "wordcell-metadata-v1", claims: select(claims), warnings: [...warnings] }); +} diff --git a/src/clip/persist.ts b/src/clip/persist.ts index d7503270..eef6ba23 100644 --- a/src/clip/persist.ts +++ b/src/clip/persist.ts @@ -1,3 +1,4 @@ +import { parseArticleMetadata, type ArticleMetadata } from "./metadata.js"; import { chmodSync, existsSync, @@ -48,6 +49,7 @@ export type VideoContextManifestMetadata = { }; export type CaptureManifestInput = { + readonly metadata?: ArticleMetadata; readonly sourceUrl: string; readonly canonicalUrl: string; readonly capturedAt: string; @@ -1248,6 +1250,7 @@ function normalizeManifest( return { schemaVersion: CAPTURE_MANIFEST_SCHEMA_VERSION, document, + ...(input.metadata === undefined ? {} : { metadata: parseArticleMetadata(input.metadata, redactSensitiveText) }), sourceUrl: sanitizeArtifactUrl(input.sourceUrl), canonicalUrl: sanitizeArtifactUrl(input.canonicalUrl), capturedAt: new Date(capturedAtTime).toISOString(), diff --git a/src/pdf.ts b/src/pdf.ts index ef0c92c9..09267197 100644 --- a/src/pdf.ts +++ b/src/pdf.ts @@ -90,3 +90,13 @@ export type { PdfToolPaths, PdfToolRunner, } from "./pdf/model.js"; + +import { inspectPdfCaptureBundle as inspectPdfCaptureBundleImplementation, verifyPdfCaptureBundle as verifyPdfCaptureBundleImplementation } from "./pdf/bundle-reader.js"; +import { PdfCaptureError as PdfCaptureErrorImplementation } from "./pdf/errors.js"; +export const inspectPdfCaptureBundle: typeof inspectPdfCaptureBundleImplementation = (...arguments_) => inspectPdfCaptureBundleImplementation(...arguments_); +export const verifyPdfCaptureBundle: typeof verifyPdfCaptureBundleImplementation = (...arguments_) => verifyPdfCaptureBundleImplementation(...arguments_); +export const PdfCaptureError = PdfCaptureErrorImplementation; +export type PdfCaptureError = InstanceType; +export type { PdfCaptureErrorCode } from "./pdf/errors.js"; +export type { PdfBundleInspection, PdfBundleVerification, ReadPdfCaptureBundleOptions, PdfBundleIntegrity } from "./pdf/bundle-reader.js"; +export type { PdfQualityIssue } from "./pdf/model.js"; diff --git a/src/pdf/bundle-reader.test.ts b/src/pdf/bundle-reader.test.ts new file mode 100644 index 00000000..74be01b8 --- /dev/null +++ b/src/pdf/bundle-reader.test.ts @@ -0,0 +1,87 @@ +import { afterEach, expect, test } from "bun:test"; +import fc from "fast-check"; +import { createHash } from "node:crypto"; +import { mkdtempSync, realpathSync, writeFileSync, mkdirSync, rmSync, symlinkSync, linkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { inspectPdfCaptureBundle, verifyPdfCaptureBundle } from "./bundle-reader.js"; +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); +const hash = (value: string): string => createHash("sha256").update(value).digest("hex"); +function fixture() { + const root = realpathSync(mkdtempSync(join(tmpdir(), "pdf-bundle-test-"))); roots.push(root); + const source = "%PDF-1.4\n%%EOF\n", markdown = "# Sample\n\n\n\n![pixel](assets/image.png)\n", image = "fixture"; + mkdirSync(join(root, "assets")); + writeFileSync(join(root, "source.pdf"), source); writeFileSync(join(root, "note.md"), markdown); writeFileSync(join(root, "assets/image.png"), image); + const descriptor = (path: string, text: string) => ({ path, bytes: Buffer.byteLength(text), sha256: hash(text) }); + const manifest = { schemaVersion: 1, kind: "pdf", status: "complete", document: { pageCount: 1, processedPages: 1 }, source: { ...descriptor("source.pdf", source), mimeType: "application/pdf" }, markdown: descriptor("note.md", markdown), images: [{ id: "image-1", page: 1, asset: descriptor("assets/image.png", image) }], annotations: null, extraction: { imageCount: 1 } }; + const save = (value: unknown = manifest) => writeFileSync(join(root, "capture.json"), JSON.stringify(value)); save(); + return { root, manifest, save }; +} +test("verifies source, Markdown, image digests and page references", async () => { + const { root } = fixture(); const result = await verifyPdfCaptureBundle(root); + expect(result.ok).toBe(true); expect(result.inspection.assets).toHaveLength(1); +}); +test("legacy Markdown is readable but explicitly unverified", async () => { + const { root, manifest, save } = fixture(); const { markdown: _, ...legacy } = manifest; save(legacy); + const result = await verifyPdfCaptureBundle(root); expect(result.ok).toBe(false); expect(result.inspection.markdown.integrity).toBe("unavailable"); +}); +test("tampering never verifies", async () => { + const { root } = fixture(); writeFileSync(join(root, "source.pdf"), "%PDF-tampered"); writeFileSync(join(root, "assets/image.png"), "damage"); + const result = await verifyPdfCaptureBundle(root); expect(result.ok).toBe(false); expect(result.issues.map(issue => issue.kind)).toEqual(["source-integrity", "asset-integrity"]); +}); +test("rejects unsupported schemas and false complete quality", async () => { + const { root, manifest, save } = fixture(); save({ ...manifest, schemaVersion: 2 }); await expect(inspectPdfCaptureBundle(root)).rejects.toMatchObject({ code: "unsupported-schema" }); + save({ ...manifest, quality: { schemaVersion: 1, issues: [{ page: 1, code: "reading-order-uncertain" }] } }); await expect(inspectPdfCaptureBundle(root)).rejects.toThrow("cannot claim complete"); +}); +test("rejects linked files, linked parents, aliases and multiple hard links", async () => { + const { root } = fixture(); rmSync(join(root, "assets/image.png")); symlinkSync(join(root, "source.pdf"), join(root, "assets/image.png")); await expect(inspectPdfCaptureBundle(root)).rejects.toThrow("single-link"); + rmSync(join(root, "assets/image.png")); linkSync(join(root, "source.pdf"), join(root, "assets/image.png")); await expect(inspectPdfCaptureBundle(root)).rejects.toThrow("single-link"); + rmSync(join(root, "assets"), { recursive: true }); symlinkSync(root, join(root, "assets")); await expect(inspectPdfCaptureBundle(root)).rejects.toThrow("ancestor"); + const alias = join(root, "alias"); symlinkSync(root, alias); await expect(inspectPdfCaptureBundle(alias)).rejects.toThrow("canonical"); +}); +test("rejects invalid UTF-8, image page references and asset quotas", async () => { + const { root, manifest, save } = fixture(); + await expect(inspectPdfCaptureBundle(root, { maxTotalAssetBytes: 2 })).rejects.toMatchObject({ code: "limit-exceeded" }); + manifest.images[0]!.page = 2; save(); await expect(inspectPdfCaptureBundle(root)).rejects.toThrow("page reference"); + manifest.images[0]!.page = 1; save(); writeFileSync(join(root, "note.md"), Buffer.from([0xff])); await expect(inspectPdfCaptureBundle(root)).rejects.toThrow("UTF-8"); +}); +test("rejects mismatched annotation references", async () => { + const { root, manifest, save } = fixture(); const bytes = JSON.stringify([{ id: "other", sha256: "a".repeat(64), kind: "visual" }]); + writeFileSync(join(root, "annotations.json"), bytes); + save({ ...manifest, annotations: { path: "annotations.json", bytes: Buffer.byteLength(bytes), sha256: hash(bytes), count: 1 } }); + await expect(inspectPdfCaptureBundle(root)).rejects.toThrow("unknown image"); +}); +test("arbitrary traversal paths are rejected", async () => { + const { root, manifest, save } = fixture(); + await fc.assert(fc.asyncProperty(fc.stringMatching(/^[a-z]{1,20}$/), async value => { + save({ ...manifest, markdown: { ...manifest.markdown, path: `../${value}.md` } }); + await expect(inspectPdfCaptureBundle(root)).rejects.toThrow("unsafe segment"); + }), { numRuns: 50 }); +}); +test("undeclared Markdown assets cannot pass verification", async () => { + const {root, manifest, save}=fixture(); + const markdown="\n![one](assets/image.png) ![missing](assets/absent.png)\n"; + writeFileSync(join(root,"note.md"),markdown);manifest.markdown={path:"note.md",bytes:Buffer.byteLength(markdown),sha256:hash(markdown)};save(); + await expect(verifyPdfCaptureBundle(root)).rejects.toThrow("undeclared PDF asset"); +}); +test("cumulative asset budget accounts actual bytes when declared sizes are understated", async () => { + const {root,manifest,save}=fixture(); + const markdown="\n![one](assets/image.png) ![two](assets/second.png)\n"; + writeFileSync(join(root,"note.md"),markdown);writeFileSync(join(root,"assets/second.png"),"fixture"); + manifest.markdown={path:"note.md",bytes:Buffer.byteLength(markdown),sha256:hash(markdown)}; + manifest.images[0]!.asset.bytes=1;manifest.images.push({id:"image-2",page:1,asset:{path:"assets/second.png",bytes:1,sha256:hash("fixture")}});manifest.extraction.imageCount=2;save(); + await expect(inspectPdfCaptureBundle(root,{maxTotalAssetBytes:10})).rejects.toThrow("exceeds"); + const result=await verifyPdfCaptureBundle(root,{maxTotalAssetBytes:14});expect(result.ok).toBe(false);expect(result.inspection.assets).toHaveLength(2); +}); +test("invalid annotation shapes have a typed invalid-input failure",async()=>{ + const {root,manifest,save}=fixture();const bytes=JSON.stringify([{kind:"unknown"}]);writeFileSync(join(root,"annotations.json"),bytes); + save({...manifest,annotations:{path:"annotations.json",bytes:Buffer.byteLength(bytes),sha256:hash(bytes),count:1}}); + await expect(inspectPdfCaptureBundle(root)).rejects.toMatchObject({code:"invalid-input"}); +}); +test("Markdown cannot silently associate a retained image with an undeclared page",async()=>{ + const {root,manifest,save}=fixture();const markdown="\n![one](assets/image.png)\n\n![two](assets/image.png)\n"; + writeFileSync(join(root,"note.md"),markdown);manifest.document={pageCount:2,processedPages:2};manifest.markdown={path:"note.md",bytes:Buffer.byteLength(markdown),sha256:hash(markdown)};save(); + await expect(verifyPdfCaptureBundle(root)).rejects.toThrow("page disagrees"); + manifest.images.push({...manifest.images[0]!,id:"image-2",page:2});manifest.extraction.imageCount=2;save();expect((await verifyPdfCaptureBundle(root)).ok).toBe(true); +}); diff --git a/src/pdf/bundle-reader.ts b/src/pdf/bundle-reader.ts new file mode 100644 index 00000000..b8a37fea --- /dev/null +++ b/src/pdf/bundle-reader.ts @@ -0,0 +1,457 @@ +import { createHash } from "node:crypto"; +import { constants as fsConstants, type BigIntStats } from "node:fs"; +import { lstat, open, opendir, realpath, type FileHandle } from "node:fs/promises"; +import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; +import { PdfCaptureError } from "./errors.js"; +import { parsePdfImageInterpretations } from "./layout.js"; + +// The same no-follow/inode fencing used by the article reader, specialized for PDF. +function boundedInteger(value: number | undefined, fallback: number, maximum: number, label: string): number { + const selected = value ?? fallback; + if (!Number.isSafeInteger(selected) || selected < 1 || selected > maximum) { + throw new PdfCaptureError("invalid-input", `${label} must be an integer from 1 through ${maximum}`); + } + return selected; +} + +function confinedRelativePath(value: unknown, label: string): string { + if (typeof value !== "string" || value.length === 0 || value.length > 4_096 || value.includes("\0")) { + throw new PdfCaptureError("integrity", `${label} must be a bounded non-empty path`); + } + if (isAbsolute(value) || value.includes("\\")) throw new PdfCaptureError("integrity", `${label} must be a relative POSIX path`); + const segments = value.split("/"); + if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) { + throw new PdfCaptureError("integrity", `${label} contains an unsafe segment`); + } + return value; +} + +type SafeBundleRoot = { + readonly path: string; + readonly handle: FileHandle; + readonly device: bigint; + readonly inode: bigint; +}; + +class MissingCaptureAssetError extends PdfCaptureError { + constructor(message: string) { super("integrity", message); } +} + +function isMissingPathError(error: unknown): boolean { + return error instanceof Error && "code" in error && error.code === "ENOENT"; +} + +function assertAssetVerificationDeadline(deadline: number | undefined): void { + if (deadline !== undefined && performance.now() >= deadline) { + throw new PdfCaptureError("timeout", "capture asset verification exceeded its time budget"); + } +} + +function confined(root: SafeBundleRoot, relativePath: string, label: string): string { + const path = resolve(root.path, relativePath); + const lexical = relative(root.path, path); + if (lexical === "" || lexical === ".." || lexical.startsWith(`..${sep}`) || isAbsolute(lexical)) { + throw new PdfCaptureError("integrity", `${label} escapes the capture bundle`); + } + return path; +} + +async function safeDirectory(pathInput: string): Promise { + if (pathInput.trim() === "") throw new PdfCaptureError("integrity", "capture bundle path is required"); + const path = resolve(pathInput); + const canonical = await realpath(path); + if (canonical !== path) throw new PdfCaptureError("integrity", "capture bundle path must be canonical and cannot traverse a symbolic link"); + const stats = await lstat(path, { bigint: true }); + if (!stats.isDirectory() || stats.isSymbolicLink()) throw new PdfCaptureError("integrity", "capture bundle path must be a real directory"); + if (dirname(path) === path) throw new PdfCaptureError("integrity", "refusing a filesystem root as a capture bundle"); + const handle = await open(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + try { + const opened = await handle.stat({ bigint: true }); + if (!opened.isDirectory() || opened.dev !== stats.dev || opened.ino !== stats.ino) { + throw new PdfCaptureError("integrity", "capture bundle root changed while it was opened"); + } + return { path, handle, device: opened.dev, inode: opened.ino }; + } catch (error) { + await handle.close(); + throw error; + } +} + +async function assertBundleRoot(root: SafeBundleRoot, deadline?: number): Promise { + assertAssetVerificationDeadline(deadline); + const [opened, named, canonical] = await Promise.all([ + root.handle.stat({ bigint: true }), + lstat(root.path, { bigint: true }), + realpath(root.path), + ]); + assertAssetVerificationDeadline(deadline); + if ( + canonical !== root.path + || !opened.isDirectory() + || !named.isDirectory() + || named.isSymbolicLink() + || opened.dev !== root.device + || opened.ino !== root.inode + || named.dev !== root.device + || named.ino !== root.inode + ) throw new PdfCaptureError("integrity", "capture bundle root changed during inspection"); +} + +async function assertAncestorChain( + root: SafeBundleRoot, + relativePath: string, + label: string, + deadline?: number, +): Promise { + await assertBundleRoot(root, deadline); + const segments = confinedRelativePath(relativePath, label).split("/"); + let current = root.path; + for (const segment of segments.slice(0, -1)) { + current = resolve(current, segment); + assertAssetVerificationDeadline(deadline); + const metadata = await lstat(current, { bigint: true }); + assertAssetVerificationDeadline(deadline); + if (!metadata.isDirectory() || metadata.isSymbolicLink()) { + throw new PdfCaptureError("integrity", `${label} must not traverse a linked or non-directory ancestor`); + } + const canonical = await realpath(current); + assertAssetVerificationDeadline(deadline); + if (canonical !== current) { + throw new PdfCaptureError("integrity", `${label} must not traverse a filesystem alias`); + } + } + await assertBundleRoot(root, deadline); +} + +async function withRegularFile( + root: SafeBundleRoot, + relativePath: string, + maximumBytes: number, + label: string, + read: (handle: FileHandle, size: number) => Promise, + deadline?: number, + missingError?: Error, +): Promise { + await assertAncestorChain(root, relativePath, label, deadline); + const path = confined(root, relativePath, label); + assertAssetVerificationDeadline(deadline); + let before: BigIntStats; + try { + before = await lstat(path, { bigint: true }); + } catch (error) { + assertAssetVerificationDeadline(deadline); + if (missingError !== undefined && isMissingPathError(error)) throw missingError; + throw error; + } + assertAssetVerificationDeadline(deadline); + if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1n) { + throw new PdfCaptureError("integrity", `${label} must be a regular single-link file`); + } + if (before.size > BigInt(maximumBytes)) throw new PdfCaptureError("integrity", `${label} exceeds ${maximumBytes} bytes`); + const canonical = await realpath(path); + assertAssetVerificationDeadline(deadline); + if (canonical !== path) throw new PdfCaptureError("integrity", `${label} must not traverse a filesystem alias`); + const handle = await open(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + try { + assertAssetVerificationDeadline(deadline); + const opened = await handle.stat({ bigint: true }); + assertAssetVerificationDeadline(deadline); + if ( + !opened.isFile() + || opened.nlink !== 1n + || opened.dev !== before.dev + || opened.ino !== before.ino + || opened.size !== before.size + ) throw new PdfCaptureError("integrity", `${label} changed while it was opened`); + await assertAncestorChain(root, relativePath, label, deadline); + const result = await read(handle, Number(opened.size)); + assertAssetVerificationDeadline(deadline); + const after = await handle.stat({ bigint: true }); + assertAssetVerificationDeadline(deadline); + const named = await lstat(path, { bigint: true }); + assertAssetVerificationDeadline(deadline); + await assertAncestorChain(root, relativePath, label, deadline); + const finalCanonical = await realpath(path); + assertAssetVerificationDeadline(deadline); + if ( + after.dev !== before.dev + || after.ino !== before.ino + || after.size !== before.size + || after.mtimeNs !== before.mtimeNs + || after.nlink !== 1n + || named.dev !== before.dev + || named.ino !== before.ino + || named.size !== before.size + || named.mtimeNs !== before.mtimeNs + || named.nlink !== 1n + || named.isSymbolicLink() + || finalCanonical !== path + ) throw new PdfCaptureError("integrity", `${label} changed while it was read`); + return result; + } finally { + await handle.close(); + assertAssetVerificationDeadline(deadline); + } +} + +async function readRegularFile( + root: SafeBundleRoot, + relativePath: string, + maximumBytes: number, + label: string, + deadline?: number, +): Promise { + return withRegularFile(root, relativePath, maximumBytes, label, async (handle, size) => { + const bytes = Buffer.alloc(size); + let offset = 0; + while (offset < bytes.byteLength) { + assertAssetVerificationDeadline(deadline); + const result = await handle.read(bytes, offset, bytes.byteLength - offset, offset); + if (result.bytesRead === 0) throw new PdfCaptureError("integrity", `${label} changed while it was read`); + offset += result.bytesRead; + } + return bytes; + }, deadline); +} + +async function hashRegularFile( + root: SafeBundleRoot, + relativePath: string, + maximumBytes: number, + deadline: number, + label: string, +): Promise<{ readonly bytes: number; readonly sha256: string }> { + assertAssetVerificationDeadline(deadline); + await assertBundleRoot(root, deadline); + assertAssetVerificationDeadline(deadline); + await assertAncestorChain(root, relativePath, label, deadline); + const path = confined(root, relativePath, label); + assertAssetVerificationDeadline(deadline); + try { + await lstat(path, { bigint: true }); + } catch (error) { + assertAssetVerificationDeadline(deadline); + if (isMissingPathError(error)) { + await assertBundleRoot(root, deadline); + throw new MissingCaptureAssetError(`capture asset ${relativePath} is missing`); + } + throw error; + } + assertAssetVerificationDeadline(deadline); + return withRegularFile(root, relativePath, maximumBytes, label, async (handle, size) => { + assertAssetVerificationDeadline(deadline); + const hash = createHash("sha256"); + const chunk = Buffer.allocUnsafe(Math.min(1 * 1_024 * 1_024, Math.max(1, size))); + let offset = 0; + while (offset < size) { + assertAssetVerificationDeadline(deadline); + const result = await handle.read(chunk, 0, Math.min(chunk.byteLength, size - offset), offset); + assertAssetVerificationDeadline(deadline); + if (result.bytesRead === 0) throw new PdfCaptureError("integrity", `${label} changed while it was read`); + hash.update(chunk.subarray(0, result.bytesRead)); + offset += result.bytesRead; + } + assertAssetVerificationDeadline(deadline); + return { bytes: size, sha256: hash.digest("hex") }; + }, deadline, new MissingCaptureAssetError(`capture asset ${relativePath} is missing`)); +} + +function digest(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + + +type RecordValue = Readonly>; +type Descriptor = { readonly path: string; readonly bytes: number; readonly sha256: string }; +export type PdfBundleIntegrity = "verified" | "mismatch" | "unavailable"; +export type PdfBundleFile = Descriptor & { readonly integrity: PdfBundleIntegrity }; +export type PdfBundleInspection = { + readonly root: string; + readonly schemaVersion: 1; + readonly status: "complete" | "partial"; + readonly pageCount: number; + readonly processedPages: number; + readonly markdown: PdfBundleFile & { readonly markdown: string }; + readonly source: PdfBundleFile; + readonly assets: readonly PdfBundleFile[]; + readonly annotations: PdfBundleFile | null; +}; +export type ReadPdfCaptureBundleOptions = { + readonly maxDocumentBytes?: number; + readonly maxSourceBytes?: number; + readonly maxAssetBytes?: number; + readonly maxTotalAssetBytes?: number; + readonly maxVerifiedAssets?: number; + readonly maxVerificationMs?: number; +}; +export type PdfBundleVerification = { + readonly ok: boolean; + readonly inspection: PdfBundleInspection; + readonly issues: readonly { readonly kind: "document-integrity" | "source-integrity" | "asset-integrity" | "annotation-integrity"; readonly path: string; readonly integrity: PdfBundleIntegrity }[]; +}; +function invalid(message: string): never { throw new PdfCaptureError("invalid-input", message); } +function object(value: unknown): RecordValue { + if (value === null || typeof value !== "object" || Array.isArray(value)) invalid("PDF bundle requires an object"); + return value as RecordValue; +} +function integer(value: unknown, maximum: number, minimum = 0): number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < minimum || value > maximum) invalid("PDF bundle has an invalid count or page reference"); + return value; +} +function descriptor(value: unknown): Descriptor { + const entry = object(value); + if (typeof entry.sha256 !== "string" || !/^[a-f0-9]{64}$/u.test(entry.sha256)) invalid("PDF bundle has an invalid digest"); + return { path: confinedRelativePath(entry.path, "PDF artifact"), bytes: integer(entry.bytes, 8 * 1024 ** 3), sha256: entry.sha256 }; +} +function decode(bytes: Buffer): string { + try { return new TextDecoder("utf-8", { fatal: true }).decode(bytes); } + catch { return invalid("PDF bundle text is not UTF-8"); } +} +function parse(bytes: Buffer): unknown { + try { return JSON.parse(decode(bytes)) as unknown; } + catch (error) { if (error instanceof PdfCaptureError) throw error; return invalid("PDF bundle JSON is invalid"); } +} + +/** Inspect retained source and assets without following aliases or trusting manifest paths. */ +export async function inspectPdfCaptureBundle(path: string, options: ReadPdfCaptureBundleOptions = {}): Promise { + const maxDocument = boundedInteger(options.maxDocumentBytes, 64 * 1024 ** 2, 256 * 1024 ** 2, "maxDocumentBytes"); + const maxSource = boundedInteger(options.maxSourceBytes, 512 * 1024 ** 2, 8 * 1024 ** 3, "maxSourceBytes"); + const maxAsset = boundedInteger(options.maxAssetBytes, 100 * 1024 ** 2, 8 * 1024 ** 3, "maxAssetBytes"); + const maxTotal = boundedInteger(options.maxTotalAssetBytes, 500 * 1024 ** 2, 8 * 1024 ** 3, "maxTotalAssetBytes"); + const maxAssets = boundedInteger(options.maxVerifiedAssets, 1000, 10_000, "maxVerifiedAssets"); + const deadline = performance.now() + boundedInteger(options.maxVerificationMs, 30_000, 300_000, "maxVerificationMs"); + const root = await safeDirectory(path); + try { + const manifest = object(parse(await readRegularFile(root, "capture.json", 4 * 1024 ** 2, "PDF manifest", deadline))); + if (manifest.schemaVersion !== 1) throw new PdfCaptureError("unsupported-schema", "Unsupported PDF manifest schema"); + if (manifest.kind !== "pdf" || (manifest.status !== "complete" && manifest.status !== "partial")) invalid("Invalid PDF manifest kind or status"); + const document = object(manifest.document); + const pageCount = integer(document.pageCount, 100_000, 1); + const processedPages = integer(document.processedPages, pageCount, 1); + if (processedPages < pageCount && manifest.status !== "partial") invalid("Truncated PDF cannot claim complete"); + if (manifest.quality !== undefined) { + const quality = object(manifest.quality); + if (quality.schemaVersion !== 1) throw new PdfCaptureError("unsupported-schema", "Unsupported PDF quality schema"); + if (!Array.isArray(quality.issues) || quality.issues.length > 100_000) invalid("Invalid PDF quality issues"); + for (const raw of quality.issues as unknown[]) { + const issue = object(raw); + if (!["reading-order-uncertain", "table-structure-uncertain", "math-structure-uncertain", "ocr-needed"].includes(String(issue.code))) invalid("Unknown PDF quality issue"); + integer(issue.page, processedPages, 1); + } + if (quality.issues.length > 0 && manifest.status !== "partial") invalid("Uncertain PDF cannot claim complete"); + } + const sourceDescriptor = descriptor(manifest.source); + if (sourceDescriptor.path !== "source.pdf" || object(manifest.source).mimeType !== "application/pdf") invalid("Invalid retained PDF source"); + const check = async (expected: Descriptor, maximum: number): Promise => { + if (expected.bytes > maximum) throw new PdfCaptureError("limit-exceeded", "PDF artifact exceeds verification byte limit"); + const actual = await hashRegularFile(root, expected.path, maximum, deadline, "PDF artifact"); + return { ...actual, path: expected.path, integrity: actual.bytes === expected.bytes && actual.sha256 === expected.sha256 ? "verified" : "mismatch" }; + }; + const source = await check(sourceDescriptor, maxSource); + let expectedMarkdown: Descriptor | null = null; + let markdownPath: string; + if (manifest.markdown !== undefined) { + expectedMarkdown = descriptor(manifest.markdown); + markdownPath = expectedMarkdown.path; + if (markdownPath.includes("/") || !markdownPath.endsWith(".md")) invalid("Invalid PDF Markdown path"); + } else { + // Legacy bundles retain their readable Markdown, but its digest is not attested. + const directory = await opendir(root.path); + const candidates: string[] = []; + let entries = 0; + try { + for await (const entry of directory) { + assertAssetVerificationDeadline(deadline); + if (++entries > 1000) throw new PdfCaptureError("limit-exceeded", "Legacy PDF directory exceeds entry limit"); + if (entry.name.endsWith(".md")) candidates.push(entry.name); + } + } finally { try { await directory.close(); } catch { /* Iteration closes the directory. */ } } + if (candidates.length !== 1) invalid("Legacy PDF bundle must contain exactly one Markdown document"); + markdownPath = candidates[0]!; + } + const markdownBytes = await readRegularFile(root, markdownPath, maxDocument, "PDF Markdown", deadline); + const markdownText = decode(markdownBytes); + const markdownDigest = digest(markdownBytes); + const markdown: PdfBundleInspection["markdown"] = { + path: markdownPath, bytes: markdownBytes.length, sha256: markdownDigest, markdown: markdownText, + integrity: expectedMarkdown === null ? "unavailable" : expectedMarkdown.bytes === markdownBytes.length && expectedMarkdown.sha256 === markdownDigest ? "verified" : "mismatch", + }; + const assetReferences = new Map>(); + let currentPage = 0; + for (const match of markdownText.matchAll(/|!\[[^\]\n]*\]\((assets\/[^)\r\n]+)\)/gu)) { + if (match[1] !== undefined) currentPage = integer(Number(match[1]), processedPages, 1); + else if (match[2] !== undefined) { + const pages = assetReferences.get(match[2]) ?? new Set(); pages.add(currentPage); assetReferences.set(match[2], pages); + } + } + const anchors = new Set(); + let anchorCount = 0; + for (const match of markdownText.matchAll(//gu)) { + if (++anchorCount > 100_000) invalid("Too many PDF page anchors"); + anchors.add(integer(Number(match[1]), processedPages, 1)); + } + if (!Array.isArray(manifest.images) || manifest.images.length > maxAssets) invalid("Invalid PDF image count"); + const imageIds = new Map(); + const assetsByPath = new Map(); + const declaredPages = new Map>(); + let totalBytes = 0; + for (const raw of manifest.images as unknown[]) { + const image = object(raw); + if (typeof image.id !== "string" || !/^[a-z0-9][a-z0-9._:-]{0,255}$/u.test(image.id) || imageIds.has(image.id)) invalid("Invalid or duplicate PDF image ID"); + const page = integer(image.page, processedPages, 1); + if (!anchors.has(page)) invalid("PDF image has no corresponding Markdown page anchor"); + const asset = descriptor(image.asset); + if (!/^assets\/[^/]+$/u.test(asset.path)) invalid("PDF image must be a direct assets child"); + if (!assetReferences.get(asset.path)?.has(page)) invalid("PDF image has no corresponding Markdown asset reference"); + imageIds.set(image.id, asset.sha256); + const pages = declaredPages.get(asset.path) ?? new Set(); + pages.add(page); declaredPages.set(asset.path, pages); + const existing = assetsByPath.get(asset.path); + if (existing !== undefined) { + if (existing.bytes !== asset.bytes || existing.sha256 !== asset.sha256) invalid("Conflicting PDF asset descriptors"); + } else { + totalBytes += asset.bytes; + if (totalBytes > maxTotal) throw new PdfCaptureError("limit-exceeded", "PDF assets exceed total byte limit"); + const checked = await check(asset, Math.min(maxAsset, maxTotal - (totalBytes - asset.bytes))); + totalBytes += checked.bytes - asset.bytes; + assetsByPath.set(asset.path, checked); + } + } + for (const [path, pages] of assetReferences) { + if (!assetsByPath.has(path)) invalid("Markdown references an undeclared PDF asset"); + if ([...pages].some(page => !declaredPages.get(path)?.has(page))) invalid("Markdown asset page disagrees with PDF manifest"); + } + const extraction = object(manifest.extraction); + if (integer(extraction.imageCount, maxAssets) !== imageIds.size) invalid("PDF image count disagrees with manifest"); + let annotations: PdfBundleFile | null = null; + if (manifest.annotations !== null) { + const annotationDescriptor = descriptor(manifest.annotations); + if (annotationDescriptor.path !== "annotations.json") invalid("Invalid PDF annotations path"); + const annotationBytes = await readRegularFile(root, "annotations.json", 16 * 1024 ** 2, "PDF annotations", deadline); + const annotationHash = digest(annotationBytes); + annotations = { path: annotationDescriptor.path, bytes: annotationBytes.length, sha256: annotationHash, integrity: annotationDescriptor.bytes === annotationBytes.length && annotationDescriptor.sha256 === annotationHash ? "verified" : "mismatch" }; + let retained: ReturnType; + try { retained = parsePdfImageInterpretations(parse(annotationBytes)); } + catch (error) { if (error instanceof PdfCaptureError) throw error; throw new PdfCaptureError("invalid-input", "Invalid PDF image annotations", { cause: error }); } + if (retained.length !== integer(object(manifest.annotations).count, 10_000)) invalid("PDF annotation count disagrees with manifest"); + for (const annotation of retained) { + if (imageIds.get(annotation.id) !== annotation.sha256) invalid("PDF annotation references an unknown image or digest"); + } + } + await assertBundleRoot(root, deadline); + return { root: root.path, schemaVersion: 1, status: manifest.status, pageCount, processedPages, markdown, source, assets: [...assetsByPath.values()], annotations }; + } finally { await root.handle.close(); } +} + +/** Missing legacy Markdown provenance is explicit and never counted as verified. */ +export async function verifyPdfCaptureBundle(path: string, options: ReadPdfCaptureBundleOptions = {}): Promise { + const inspection = await inspectPdfCaptureBundle(path, options); + const issues: Array = []; + const add = (file: PdfBundleFile, kind: PdfBundleVerification["issues"][number]["kind"]): void => { + if (file.integrity !== "verified") issues.push({ kind, path: file.path, integrity: file.integrity }); + }; + add(inspection.markdown, "document-integrity"); add(inspection.source, "source-integrity"); + for (const asset of inspection.assets) add(asset, "asset-integrity"); + if (inspection.annotations !== null) add(inspection.annotations, "annotation-integrity"); + return { ok: issues.length === 0, inspection, issues }; +} diff --git a/src/pdf/capture.test.ts b/src/pdf/capture.test.ts index eea248c0..228f2abe 100644 --- a/src/pdf/capture.test.ts +++ b/src/pdf/capture.test.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import { mkdtempSync, readFileSync, + realpathSync, rmSync, writeFileSync, } from "node:fs"; @@ -10,6 +11,7 @@ import { basename, dirname, join } from "node:path"; import { afterEach, describe, expect, test } from "bun:test"; +import { verifyPdfCaptureBundle } from "./bundle-reader.js"; import { runPdfCapture } from "./capture.js"; import type { PdfCaptureDependencies, @@ -54,6 +56,7 @@ function fakeDependencies( options: { readonly failOnTesseract?: boolean; readonly imageCount?: number; + readonly uncertainMath?: boolean; } = {}, ): PdfCaptureDependencies { return { @@ -92,7 +95,7 @@ function fakeDependencies( Fixture PDF -A useful paragraph. +${options.uncertainMath ? "∫ f(x)" : "A useful paragraph."} ${images.join("\n")} `); @@ -228,3 +231,15 @@ describe("PDF capture", () => { expect(outcome.warnings.join(" ")).toContain("configured asset byte limit was reached"); }); }); + + +test("capture returns the exact persisted Markdown digest and partial quality", async () => { + const fixture = sourceFixture(); + const outcome = await runPdfCapture({ inputPath: fixture.sourcePath, outputBase: fixture.outputBase }, fakeDependencies({ uncertainMath: true })); + expect(outcome.status).toBe("partial"); + expect(outcome.manifest.quality?.issues).toEqual([{ page: 1, code: "math-structure-uncertain" }]); + const bytes = readFileSync(outcome.markdownPath); + expect(outcome.manifest.markdown?.sha256).toBe(createHash("sha256").update(bytes).digest("hex")); + expect(outcome.manifest.markdown?.bytes).toBe(bytes.length); + expect((await verifyPdfCaptureBundle(realpathSync(outcome.outputDirectory))).ok).toBe(true); +}); diff --git a/src/pdf/capture.ts b/src/pdf/capture.ts index 0f73fd5b..991524da 100644 --- a/src/pdf/capture.ts +++ b/src/pdf/capture.ts @@ -8,7 +8,7 @@ import { slugify } from "../clip/lib.js"; import { sanitizeTerminalText } from "../clip/terminal.js"; import { inspectPdf } from "./extract.js"; import { - layoutBlocks, + analyzePdfLayout, parsePdfImageInterpretations, } from "./layout.js"; import { @@ -183,6 +183,8 @@ export async function runPdfCapture( }); } const platforms = embeddedPlatforms(resolvedImages); + const layout = analyzePdfLayout(inspection.pages); + for (const issue of layout.issues) warnings.push(`Page ${issue.page}: ${issue.code}; consult the retained source PDF.`); const status = warnings.length > 0 || unclassifiedImages > 0 || ocrWarningCount > 0 ? "partial" : "complete"; @@ -190,7 +192,7 @@ export async function runPdfCapture( const requestedSlug = options.slug ?? inspection.metadata.title ?? filenameTitle; const slug = slugify(sanitizeTerminalText(requestedSlug)); if (slug === "") throw new Error("could not derive a safe PDF capture slug; pass an explicit slug"); - const blocks = layoutBlocks(inspection.pages); + const blocks = layout.blocks; const byId = new Map(resolvedImages.map((image) => [image.image.id, image])); const capturedAt = (dependencies.now ?? (() => new Date()))().toISOString(); const built = buildPdfMarkdown({ @@ -233,6 +235,8 @@ export async function runPdfCapture( const manifest: PdfCaptureManifest = { schemaVersion: PDF_CAPTURE_MANIFEST_SCHEMA_VERSION, kind: "pdf", + quality: { schemaVersion: 1, issues: layout.issues }, + markdown: { path: pdfMarkdownFilename(slug), bytes: Buffer.byteLength(built.markdown), sha256: createHash("sha256").update(built.markdown).digest("hex") }, capturedAt, status, source: { diff --git a/src/pdf/cli.ts b/src/pdf/cli.ts index b38150c7..5173fca6 100644 --- a/src/pdf/cli.ts +++ b/src/pdf/cli.ts @@ -1,3 +1,4 @@ +import { PdfCaptureError } from "./errors.js"; import { lstatSync, readFileSync } from "node:fs"; import { resolve } from "node:path"; @@ -149,7 +150,7 @@ export async function main( return 0; } catch (error) { const message = safe(error instanceof Error ? error.message : String(error)); - if (arguments_.json) output.stdout(terminalSafeJson({ ok: false, error: message })); + if (arguments_.json) output.stdout(terminalSafeJson({ ok: false, error: message, code: error instanceof PdfCaptureError ? error.code : "unknown" })); else output.stderr(renderFailure(message, "wordcell doctor", stderrStyle(environment, output))); return 1; } finally { diff --git a/src/pdf/errors.test.ts b/src/pdf/errors.test.ts new file mode 100644 index 00000000..8ee358ae --- /dev/null +++ b/src/pdf/errors.test.ts @@ -0,0 +1,27 @@ +import { expect, test } from "bun:test"; +import { runPdfToolCommand } from "./tools.js"; +import { PdfCaptureError } from "./errors.js"; + +test("tool timeout and output overflow preserve typed codes", async () => { + await expect(runPdfToolCommand({ command: [process.execPath, "-e", "setTimeout(()=>{},10000)"], timeoutMs: 20, maxOutputBytes: 1024 })).rejects.toMatchObject({ code: "timeout" }); + await expect(runPdfToolCommand({ command: [process.execPath, "-e", "process.stdout.write('a'.repeat(8192))"], timeoutMs: 5000, maxOutputBytes: 8 })).rejects.toMatchObject({ code: "limit-exceeded" }); +}); +test("missing executable is observed as tool-unavailable", async () => { + await expect(runPdfToolCommand({ command: ["/nonexistent/wordcell-test"], timeoutMs: 1000, maxOutputBytes: 1024 })).rejects.toMatchObject({ code: "tool-unavailable" }); +}); +test("typed error retains cause without classifying unknown prose", () => { + const cause = new Error("arbitrary private failure"); const error = new PdfCaptureError("unknown", "PDF operation failed", { cause }); + expect(error.cause).toBe(cause); expect(error.code).toBe("unknown"); +}); + +test("external supervision keeps PDF tools in the owned parent group", async () => { + if (process.platform === "win32") return; + const { execFileSync } = await import("node:child_process"); + const parentGroup = Number(execFileSync("/bin/ps", ["-o", "pgid=", "-p", String(process.pid)], { encoding: "utf8" }).trim()); + const code = "const {execFileSync}=require('node:child_process');process.stdout.write(JSON.stringify({pid:process.pid,group:Number(execFileSync('/bin/ps',['-o','pgid=','-p',String(process.pid)],{encoding:'utf8'}).trim())}))"; + const command = [process.execPath, "-e", code]; + const standalone = JSON.parse((await runPdfToolCommand({ command, timeoutMs: 5000, maxOutputBytes: 1024 })).stdout); + expect(standalone.group).toBe(standalone.pid); + const supervised = JSON.parse((await runPdfToolCommand({ command, processGroup: "inherit", timeoutMs: 5000, maxOutputBytes: 1024 })).stdout); + expect(supervised.group).toBe(parentGroup); +}); diff --git a/src/pdf/errors.ts b/src/pdf/errors.ts new file mode 100644 index 00000000..30d76d09 --- /dev/null +++ b/src/pdf/errors.ts @@ -0,0 +1,11 @@ +export type PdfCaptureErrorCode = "invalid-input" | "encrypted" | "limit-exceeded" | "tool-unavailable" | "extraction-failed" | "timeout" | "cancelled" | "integrity" | "unsupported-schema" | "unknown"; + +/** Codes describe observed failures; unknown errors are never classified from their prose. */ +export class PdfCaptureError extends Error { + readonly code: PdfCaptureErrorCode; + constructor(code: PdfCaptureErrorCode, message: string, options?: ErrorOptions) { + super(message, options); + this.code = code; + this.name = "PdfCaptureError"; + } +} diff --git a/src/pdf/extract.ts b/src/pdf/extract.ts index 2a34a6c9..262dae54 100644 --- a/src/pdf/extract.ts +++ b/src/pdf/extract.ts @@ -1,3 +1,4 @@ +import { PdfCaptureError } from "./errors.js"; import { createHash } from "node:crypto"; import { chmodSync, @@ -43,7 +44,7 @@ export const pdfCaptureDefaults = { function positiveBound(value: number | undefined, fallback: number, maximum: number, label: string): number { const selected = value ?? fallback; if (!Number.isSafeInteger(selected) || selected < 1 || selected > maximum) { - throw new Error(`${label} must be an integer from 1 through ${maximum}`); + throw new PdfCaptureError("invalid-input", `${label} must be an integer from 1 through ${maximum}`); } return selected; } @@ -61,10 +62,10 @@ function prepareWorkspace(path: string): string { if (existsSync(absolute)) { const stats = lstatSync(absolute); if (!stats.isDirectory() || stats.isSymbolicLink()) { - throw new Error("PDF inspection workspace must be a real directory"); + throw new PdfCaptureError("invalid-input", "PDF inspection workspace must be a real directory"); } if (readdirSync(absolute).length !== 0) { - throw new Error("PDF inspection workspace must be empty"); + throw new PdfCaptureError("invalid-input", "PDF inspection workspace must be empty"); } } else { mkdirSync(absolute, { recursive: false, mode: 0o700 }); @@ -81,15 +82,15 @@ function sourceIdentity(path: string, maxPdfBytes: number): { const originalFilename = basename(path); const canonical = realpathSync(resolve(path)); const stats = statSync(canonical); - if (!stats.isFile()) throw new Error("PDF input must be a regular file"); - if (stats.size < 5) throw new Error("PDF input is too small to contain a PDF header"); - if (stats.size > maxPdfBytes) throw new Error(`PDF input exceeds the ${maxPdfBytes}-byte limit`); + if (!stats.isFile()) throw new PdfCaptureError("invalid-input", "PDF input must be a regular file"); + if (stats.size < 5) throw new PdfCaptureError("invalid-input", "PDF input is too small to contain a PDF header"); + if (stats.size > maxPdfBytes) throw new PdfCaptureError("limit-exceeded", `PDF input exceeds the ${maxPdfBytes}-byte limit`); const descriptor = openSync(canonical, "r"); try { const signature = Buffer.alloc(5); const count = readSync(descriptor, signature, 0, signature.length, 0); if (count !== signature.length || signature.toString("ascii") !== "%PDF-") { - throw new Error("PDF input does not have a valid PDF signature"); + throw new PdfCaptureError("invalid-input", "PDF input does not have a valid PDF signature"); } } finally { closeSync(descriptor); @@ -155,18 +156,18 @@ function safeGeneratedImage( function generatedTreeSize(workspace: string, maximumFiles: number, maximumBytes: number): void { const entries = readdirSync(workspace, { withFileTypes: true }); if (entries.length > maximumFiles) { - throw new Error(`PDF extraction created more than ${maximumFiles} files`); + throw new PdfCaptureError("limit-exceeded", `PDF extraction created more than ${maximumFiles} files`); } let totalBytes = 0; for (const entry of entries) { const path = join(workspace, entry.name); const stats = lstatSync(path); if (!entry.isFile() || stats.isSymbolicLink()) { - throw new Error("PDF extraction created an unexpected non-file output"); + throw new PdfCaptureError("extraction-failed", "PDF extraction created an unexpected non-file output"); } totalBytes += stats.size; if (totalBytes > maximumBytes) { - throw new Error(`PDF extraction exceeded the ${maximumBytes}-byte workspace limit`); + throw new PdfCaptureError("limit-exceeded", `PDF extraction exceeded the ${maximumBytes}-byte workspace limit`); } } } @@ -208,7 +209,9 @@ export async function inspectPdf( cwd: workspaceDirectory, }); if (info.exitCode !== 0) { - throw new Error("pdfinfo could not inspect the input PDF"); + // Poppler runs in the C locale; only its exact password diagnostic establishes encryption. + const encrypted = info.stderr.split(/\r?\n/u).some(line => line.trim() === "Command Line Error: Incorrect password"); + throw new PdfCaptureError(encrypted ? "encrypted" : "extraction-failed", "pdfinfo could not inspect the input PDF"); } const metadata = parsePdfInfo(info.stdout); const processedPageLimit = Math.min(metadata.pageCount, maxPages); @@ -233,7 +236,7 @@ export async function inspectPdf( cwd: workspaceDirectory, }); if (extracted.exitCode !== 0) { - throw new Error("pdftohtml could not extract the input PDF"); + throw new PdfCaptureError("extraction-failed", "pdftohtml could not extract the input PDF"); } generatedTreeSize( workspaceDirectory, @@ -245,7 +248,7 @@ export async function inspectPdf( ); const layoutStats = statSync(layoutPath); if (!layoutStats.isFile() || layoutStats.size > pdfCaptureDefaults.maxLayoutBytes) { - throw new Error(`Poppler layout XML exceeds the ${pdfCaptureDefaults.maxLayoutBytes}-byte limit`); + throw new PdfCaptureError("limit-exceeded", `Poppler layout XML exceeds the ${pdfCaptureDefaults.maxLayoutBytes}-byte limit`); } const parsed = parsePopplerXml(readFileSync(layoutPath, "utf8"), { maxPages: processedPageLimit, diff --git a/src/pdf/layout-analysis.test.ts b/src/pdf/layout-analysis.test.ts new file mode 100644 index 00000000..f8263bc7 --- /dev/null +++ b/src/pdf/layout-analysis.test.ts @@ -0,0 +1,40 @@ +import { expect, test } from "bun:test"; +import fc from "fast-check"; +import { analyzePdfLayout } from "./layout.js"; +import type { PdfPageLayout, PdfTextFragment } from "./model.js"; +import { buildPdfMarkdown } from "./markdown.js"; +function fragment(text: string, left: number, top: number, width = 210, extra: Partial = {}): PdfTextFragment { + return { text, left, top, width, height: 14, fontSize: 12, fontId: "1", bold: false, italic: false, ...extra }; +} +const page = (text: readonly PdfTextFragment[]): PdfPageLayout => ({ page: 1, width: 600, height: 800, text, images: [] }); +const content = (result: ReturnType): string => result.blocks.map(block => block.kind === "image" ? "image" : block.kind === "table" ? block.rows.flat().join(" ") : block.text).join(" "); +test("orders columns before merging baselines and preserves a spanning heading", () => { + const text = [fragment("Heading", 30, 30, 300, { fontSize: 24 }), ...[0, 1, 2, 3].flatMap(i => [fragment(`left${i}`, 30, 100 + i * 20), fragment(`right${i}`, 330, 100 + i * 20)])]; + const result = analyzePdfLayout([page(text)]); + expect(content(result)).toBe("Heading left0 left1 left2 left3 right0 right1 right2 right3"); expect(result.issues).toEqual([]); expect(result.blocks[0]!.kind).toBe("heading"); +}); +test("aligned compact numeric table with explicit header becomes escaped Markdown", () => { + const text = [fragment("Name\\|type", 30, 80, 60, { bold: true }), fragment("Count", 250, 80, 40, { bold: true }), ...[0, 1, 2].flatMap(i => [fragment(`item${i}`, 30, 100 + i * 20, 50), fragment(String(i), 250, 100 + i * 20, 30)])]; + const result = analyzePdfLayout([page(text)]); expect(result.blocks[0]!.kind).toBe("table"); expect(result.issues).toEqual([]); + const built = buildPdfMarkdown({ slug: "table", originalFilename: "table.pdf", sourceSha256: "a".repeat(64), capturedDate: "2026-10-16", status: "complete", metadata: { title: null, author: null, subject: null, keywords: null, creator: null, producer: null, createdAt: null, modifiedAt: null, pageCount: 1, encrypted: false }, blocks: result.blocks, images: new Map(), embeddedPlatforms: [] }); + expect(built.markdown).toContain("| Name\\\\\\|type | Count |"); expect(built.markdown).toContain("| --- | --- |"); +}); +test("adjacent raised digits remain attached while unsupported math is partial", () => { + const result = analyzePdfLayout([page([fragment("x", 30, 100, 10), fragment("2", 40, 95, 6, { fontSize: 8, height: 9 })])]); expect(content(result)).toBe("x²"); + const uncertain = analyzePdfLayout([page([fragment("∫ f(x)", 30, 100)])]); expect(uncertain.issues).toEqual([{ page: 1, code: "math-structure-uncertain" }]); +}); +test("ambiguous multi-cell geometry is reported instead of asserting a table", () => { + const result = analyzePdfLayout([page([0, 1, 2].flatMap(i => [fragment(`a${i}`, 30, 100 + i * 20, 30), fragment(`b${i}`, 200, 100 + i * 20, 30), fragment(`c${i}`, 400, 100 + i * 20, 30)]))]); + expect(result.issues.some(issue => issue.code === "table-structure-uncertain")).toBe(true); +}); +test("column output is invariant to fragment input permutation and translation", () => { + fc.assert(fc.property(fc.integer({ min: 3, max: 20 }), fc.integer({ min: 0, max: 20 }), (count, shift) => { + const text = Array.from({ length: count }, (_, i) => [fragment(`L${i}`, 30 + shift, 100 + i * 20), fragment(`R${i}`, 330 + shift, 100 + i * 20)]).flat(); + expect(content(analyzePdfLayout([page(text)]))).toBe(content(analyzePdfLayout([page([...text].reverse())]))); + const output = content(analyzePdfLayout([page(text)])); for (let i = 0; i < count; i++) { expect(output.split(`L${i}`).length).toBeGreaterThan(1); expect(output.split(`R${i}`).length).toBeGreaterThan(1); } + }), { numRuns: 50 }); +}); +test("removing repeated margins preserves identical body text",()=>{ + const pages=[1,2,3].map(number=>({...page([fragment("Methods",30,20),fragment("Methods",30,200)]),page:number})); + const result=analyzePdfLayout(pages);expect(content(result)).toBe("Methods Methods Methods"); +}); diff --git a/src/pdf/layout.ts b/src/pdf/layout.ts index ea7d89ac..3ddf6d70 100644 --- a/src/pdf/layout.ts +++ b/src/pdf/layout.ts @@ -36,6 +36,7 @@ export type ParsedPopplerXml = { }; export type PdfLayoutBlock = + | { readonly kind: "table"; readonly page: number; readonly rows: readonly (readonly string[])[] } | { readonly kind: "heading"; readonly page: number; @@ -456,7 +457,7 @@ export function layoutBlocks(pages: readonly PdfPageLayout[]): readonly PdfLayou for (const page of pages) { const lines = (linesByPage.get(page.page) ?? []).filter( - (line) => !repeatedMargins.has(normalizedMarginText(line.text)), + (line) => !(line.top <= page.height * 0.06 || line.top + line.height >= page.height * 0.94) || !repeatedMargins.has(normalizedMarginText(line.text)), ); const steps = lines.slice(1).map((line, index) => Math.max(0, line.top - (lines[index]?.top ?? line.top))); @@ -491,7 +492,7 @@ export function layoutBlocks(pages: readonly PdfPageLayout[]): readonly PdfLayou const { line, index } = event; const previous = lines[index - 1]; const gapBefore = previous === undefined ? Number.POSITIVE_INFINITY : line.top - previous.top; - const fontHeading = line.fontSize > bodySize * 1.08; + const fontHeading = line.fontSize > bodySize * 1.08 && !/[=∫∑√⁰¹²³⁴⁵⁶⁷⁸⁹]/u.test(line.text); const boldHeading = line.boldRatio >= 0.78 && line.text.length <= 180 && gapBefore >= normalStep * 1.45; @@ -516,7 +517,7 @@ export function layoutBlocks(pages: readonly PdfPageLayout[]): readonly PdfLayou } flushParagraph(); } - return output.filter((block) => block.kind === "image" || block.text.trim() !== ""); + return output.filter((block) => block.kind === "image" || block.kind === "table" || block.text.trim() !== ""); } export function parsePdfImageInterpretations(value: unknown): readonly PdfImageInterpretation[] { @@ -616,3 +617,135 @@ export function parsePdfImageInterpretations(value: unknown): readonly PdfImageI } return output; } + +/** Geometry evidence is deliberately conservative: uncertain structure remains visible in quality. */ +export function analyzePdfLayout(pages: readonly PdfPageLayout[]): { readonly blocks: readonly PdfLayoutBlock[]; readonly issues: readonly import("./model.js").PdfQualityIssue[] } { + const issues: import("./model.js").PdfQualityIssue[] = []; + const blocks: PdfLayoutBlock[] = []; + const marginLines = new Map(pages.filter(page => page.text.length <= 4000).map(page => [page.page, pageLines(page)])); + const margins = repeatedMarginLines(pages, marginLines); + for (const originalInput of pages) { + const original = { ...originalInput, text: originalInput.text.filter(fragment => !(fragment.top <= originalInput.height * 0.06 || fragment.top + fragment.height >= originalInput.height * 0.94) || !margins.has(normalizedMarginText(fragment.text))) }; + const warn = (code: import("./model.js").PdfQualityIssue["code"]): void => { + if (!issues.some(issue => issue.page === original.page && issue.code === code)) issues.push({ page: original.page, code }); + }; + // Bound quadratic geometry analysis independently of the parser's byte limit. + if (original.text.length > 4000) { + warn("reading-order-uncertain"); blocks.push(...layoutBlocks([original])); continue; + } + if (original.text.some(fragment => /[∫∑√]/u.test(fragment.text))) warn("math-structure-uncertain"); + const text = [...original.text].sort((a, b) => a.top - b.top || a.left - b.left); + const consumed = new Set(); + const replacements = new Map(); + const superscripts: Record = { "0": "⁰", "1": "¹", "2": "²", "3": "³", "4": "⁴", "5": "⁵", "6": "⁶", "7": "⁷", "8": "⁸", "9": "⁹", "+": "⁺", "-": "⁻", "=": "⁼", "(": "⁽", ")": "⁾", "n": "ⁿ", "i": "ⁱ" }; + for (const small of text) { + const bases = text.filter(base => base !== small && base.fontSize >= small.fontSize * 1.3 + && small.top < base.top && small.top + small.height > base.top - base.height * 0.25 + && Math.abs(small.left - (base.left + base.width)) < base.fontSize * 0.6); + if (bases.length === 0) continue; + const base = bases[0]!; + const glyphs = [...small.text.trim()]; + if (bases.length === 1 && glyphs.length > 0 && glyphs.length <= 8 && glyphs.every(glyph => superscripts[glyph] !== undefined) && !replacements.has(base)) { + replacements.set(base, { ...base, text: base.text + glyphs.map(glyph => superscripts[glyph]).join(""), width: small.left + small.width - base.left }); + consumed.add(small); + } else warn("math-structure-uncertain"); + } + const fragments = text.filter(fragment => !consumed.has(fragment)).map(fragment => replacements.get(fragment) ?? fragment); + const page: PdfPageLayout = { ...original, text: fragments }; + const rows: PdfTextFragment[][] = []; + for (const fragment of fragments) { + const row = rows.at(-1); + if (row === undefined || Math.abs(fragment.top - row[0]!.top) > Math.max(2, fragment.height * 0.2)) rows.push([fragment]); + else row.push(fragment); + } + const cellRows = rows.map(row => { + const groups: PdfTextFragment[][] = []; + for (const fragment of [...row].sort((a, b) => a.left - b.left)) { + const prior = groups.at(-1), last = prior?.at(-1); + if (last === undefined || fragment.left - (last.left + last.width) > Math.max(12, fragment.fontSize * 1.5)) groups.push([fragment]); + else prior!.push(fragment); + } + return groups; + }); + const tables: Array<{ top: number; bottom: number; block: PdfLayoutBlock; fragments: PdfTextFragment[] }> = []; + for (let index = 0; index < cellRows.length; index++) { + const first = cellRows[index]!; + if (first.length < 2 || first.length > 12) continue; + const group = [first]; + let next = index + 1; + for (; next < cellRows.length; next++) { + const candidate = cellRows[next]!; + if (candidate.length !== first.length || candidate.some((cell, column) => Math.abs(cell[0]!.left - first[column]![0]!.left) > 4)) break; + if (candidate[0]![0]!.top - group.at(-1)![0]![0]!.top > Math.max(40, candidate[0]![0]!.height * 3)) break; + group.push(candidate); + } + const cells = group.map(row => row.map(cell => joinFragments(cell))); + const header = first.every(cell => cell.every(fragment => fragment.bold)); + const numeric = cells.slice(1).some(row => row.slice(1).some(cell => /^[-+\d$€£]/u.test(cell.trim()))); + const compact = cells.every(row => row.every(cell => cell.length <= 100)); + if (group.length >= 3 && compact && header && numeric) { + const members = group.flat(2); + tables.push({ top: Math.min(...members.map(fragment => fragment.top)), bottom: Math.max(...members.map(fragment => fragment.top + fragment.height)), block: { kind: "table", page: page.page, rows: cells }, fragments: members }); + index = next - 1; + } else if (group.length >= 3 && first.length >= 3) warn("table-structure-uncertain"); + } + const tableFragments = new Set(tables.flatMap(table => table.fragments)); + const remaining = fragments.filter(fragment => !tableFragments.has(fragment)); + const emitBand = (band: readonly PdfTextFragment[], images: readonly PdfImageCandidate[]): void => { + if (band.length === 0 && images.length === 0) return; + // Repeated gutters are detected before line grouping can join adjacent columns. + const candidates = new Set(); + for (const row of cellRows) { + for (let i = 1; i < row.length; i++) { + const left = row[i - 1]!.at(-1)!, right = row[i]![0]!; + const gap = right.left - (left.left + left.width); + if (gap >= Math.max(page.width * 0.025, left.fontSize * 1.5)) candidates.add(Math.round((left.left + left.width + right.left) / 2)); + } + } + const valid = [...candidates].filter(gutter => { + const left = band.filter(fragment => fragment.left + fragment.width <= gutter); + const right = band.filter(fragment => fragment.left >= gutter); + if (left.length + right.length !== band.length || left.length < 3 || right.length < 3) return false; + if (left.filter(fragment => fragment.width > page.width * 0.18).length < 2 || right.filter(fragment => fragment.width > page.width * 0.18).length < 2) return false; + const overlap = Math.min(Math.max(...left.map(fragment => fragment.top)), Math.max(...right.map(fragment => fragment.top))) - Math.max(Math.min(...left.map(fragment => fragment.top)), Math.min(...right.map(fragment => fragment.top))); + return overlap > Math.max(...band.map(fragment => fragment.height)) * 1.5; + }); + if (valid.length > 0) { + const gutter = valid[Math.floor(valid.length / 2)]!; + if (images.some(image => image.left < gutter && image.left + image.width > gutter)) { + warn("reading-order-uncertain"); blocks.push(...layoutBlocks([{ ...page, text: band, images }])); return; + } + for (const left of [true, false]) blocks.push(...layoutBlocks([{ ...page, text: band.filter(fragment => left ? fragment.left + fragment.width <= gutter : fragment.left >= gutter), images: images.filter(image => left ? image.left + image.width <= gutter : image.left >= gutter) }])); + } else { + if (band.length >= 6 && cellRows.filter(row => row.length >= 2 && row.some(cell => band.includes(cell[0]!))).length >= 3) warn("reading-order-uncertain"); + blocks.push(...layoutBlocks([{ ...page, text: band, images }])); + } + }; + // Spanning headings split vertical regions; tables retain their exact place. + const bodyFont = weightedBodyFontSize(pageLines(page)); + const gutterVotes = new Map(); + for (const row of cellRows) for (let i = 1; i < row.length; i++) { + const left = row[i - 1]!.at(-1)!, right = row[i]![0]!; + if (right.left - (left.left + left.width) < Math.max(12, left.fontSize * 1.5)) continue; + const gutter = Math.round((left.left + left.width + right.left) / 20) * 10; + gutterVotes.set(gutter, (gutterVotes.get(gutter) ?? 0) + 1); + } + const repeatedGutters = [...gutterVotes].filter(([, votes]) => votes >= 3).map(([gutter]) => gutter); + const spans = remaining.filter(fragment => fragment.width > page.width * 0.65 || (fragment.fontSize > bodyFont * 1.08 && repeatedGutters.some(gutter => fragment.left < gutter && fragment.left + fragment.width > gutter))); + const boundaries = [...tables.map(table => ({ top: table.top, bottom: table.bottom, table, span: null as PdfTextFragment | null })), ...spans.map(span => ({ top: span.top, bottom: span.top + span.height, table: null, span }))].sort((a, b) => a.top - b.top); + const emitted = new Set(), emittedImages = new Set(); + for (const boundary of boundaries) { + const band = remaining.filter(fragment => !emitted.has(fragment) && fragment !== boundary.span && fragment.top < boundary.top); + const images = page.images.filter(image => !emittedImages.has(image) && image.top < boundary.top); + emitBand(band, images); for (const fragment of band) emitted.add(fragment); for (const image of images) emittedImages.add(image); + if (boundary.table !== null) blocks.push(boundary.table.block); + if (boundary.span !== null) { + const span = boundary.span; + blocks.push(span.fontSize > bodyFont * 1.08 ? { kind: "heading", page: page.page, level: 2, text: span.text } : { kind: "paragraph", page: page.page, text: span.text }); + emitted.add(span); + } + } + emitBand(remaining.filter(fragment => !emitted.has(fragment)), page.images.filter(image => !emittedImages.has(image))); + } + return { blocks, issues }; +} diff --git a/src/pdf/markdown.ts b/src/pdf/markdown.ts index 751df07b..61664aa3 100644 --- a/src/pdf/markdown.ts +++ b/src/pdf/markdown.ts @@ -24,7 +24,7 @@ export type ResolvedPdfImage = { function escapeInline(value: string): string { return sanitizeTerminalText(value) .replace(/\\/gu, "\\\\") - .replace(/([`*_[\]{}<>#])/gu, "\\$1") + .replace(/([`*_[\]{}<>#|])/gu, "\\$1") .replace(/\s+/gu, " ") .trim(); } @@ -81,6 +81,12 @@ function markdownContent( lines.push(imageMarkdown(resolved), ""); continue; } + if (block.kind === "table") { + textBlockCount += 1; + const rows = block.rows.map(row => `| ${row.map(cell => escapeInline(cell)).join(" | ")} |`); + if (rows.length > 0) lines.push(rows[0]!, `| ${block.rows[0]!.map(() => "---").join(" | ")} |`, ...rows.slice(1), ""); + continue; + } if (block.kind === "heading") { headingCount += 1; lines.push(`${"#".repeat(Math.max(2, Math.min(6, block.level)))} ${escapeInline(block.text)}`, ""); diff --git a/src/pdf/model.ts b/src/pdf/model.ts index 380520be..be51df94 100644 --- a/src/pdf/model.ts +++ b/src/pdf/model.ts @@ -126,7 +126,14 @@ export type PdfManifestImage = PdfBounds & { readonly metadata: PdfImageSemanticMetadata | null; }; +export type PdfQualityIssue = { + readonly code: "reading-order-uncertain" | "table-structure-uncertain" | "math-structure-uncertain" | "ocr-needed"; + readonly page: number; +}; + export type PdfCaptureManifest = { + readonly markdown?: { readonly path: string; readonly bytes: number; readonly sha256: string }; + readonly quality?: { readonly schemaVersion: 1; readonly issues: readonly PdfQualityIssue[] }; readonly schemaVersion: typeof PDF_CAPTURE_MANIFEST_SCHEMA_VERSION; readonly kind: "pdf"; readonly capturedAt: string; @@ -204,6 +211,8 @@ export type PdfCaptureOutcome = { }; export type PdfToolCommand = { + /** An external supervisor may own the inherited group and its final cleanup. */ + readonly processGroup?: "new" | "inherit"; readonly command: readonly string[]; readonly timeoutMs: number; readonly maxOutputBytes: number; diff --git a/src/pdf/persist.ts b/src/pdf/persist.ts index c40839e8..4b3b1058 100644 --- a/src/pdf/persist.ts +++ b/src/pdf/persist.ts @@ -212,9 +212,15 @@ export function persistPdfCapture(input: PersistPdfCaptureInput): string { chmodSync(destination, 0o644); } + const markdownBytes = `${redactSensitiveText(input.markdown).trimEnd()}\n`; + const markdownDescriptor = { path: `${slug}.md`, bytes: Buffer.byteLength(markdownBytes), sha256: createHash("sha256").update(markdownBytes).digest("hex") }; + if (input.manifest.markdown !== undefined && (input.manifest.markdown.path !== markdownDescriptor.path || input.manifest.markdown.bytes !== markdownDescriptor.bytes || input.manifest.markdown.sha256 !== markdownDescriptor.sha256)) { + throw new Error("PDF Markdown does not match its manifest provenance"); + } + const manifest = { ...input.manifest, markdown: markdownDescriptor }; writeFileSync( join(stagingDirectory, `${slug}.md`), - `${redactSensitiveText(input.markdown).trimEnd()}\n`, + markdownBytes, { encoding: "utf8", flag: "wx", mode: 0o644 }, ); if (input.manifest.annotations === null) { @@ -253,7 +259,7 @@ export function persistPdfCapture(input: PersistPdfCaptureInput): string { } writeFileSync( join(stagingDirectory, PDF_CAPTURE_MANIFEST_FILENAME), - `${JSON.stringify(input.manifest, null, 2)}\n`, + `${JSON.stringify(manifest, null, 2)}\n`, { encoding: "utf8", flag: "wx", mode: 0o644 }, ); assertStagingTreeSafe(stagingDirectory); diff --git a/src/pdf/source.ts b/src/pdf/source.ts index 496537ab..51644ec0 100644 --- a/src/pdf/source.ts +++ b/src/pdf/source.ts @@ -1,3 +1,4 @@ +import { PdfCaptureError } from "./errors.js"; import { chmodSync, mkdtempSync, @@ -37,13 +38,13 @@ function parseRemoteUrl(input: string): URL | null { try { url = new URL(input); } catch { - throw new Error("PDF URL is invalid"); + throw new PdfCaptureError("invalid-input", "PDF URL is invalid"); } if (url.protocol !== "http:" && url.protocol !== "https:") { - throw new Error("PDF URL must use HTTP or HTTPS"); + throw new PdfCaptureError("invalid-input", "PDF URL must use HTTP or HTTPS"); } if (url.username !== "" || url.password !== "") { - throw new Error("PDF URL must not contain embedded credentials"); + throw new PdfCaptureError("invalid-input", "PDF URL must not contain embedded credentials"); } return url; } @@ -69,7 +70,7 @@ function assertPdfSignature(bytes: Uint8Array): void { bytes.byteLength < 5 || new TextDecoder().decode(bytes.subarray(0, 5)) !== "%PDF-" ) { - throw new Error("remote PDF input does not have a valid PDF signature"); + throw new PdfCaptureError("invalid-input", "remote PDF input does not have a valid PDF signature"); } } diff --git a/src/pdf/tools.ts b/src/pdf/tools.ts index 9af1f7c6..6f4901ad 100644 --- a/src/pdf/tools.ts +++ b/src/pdf/tools.ts @@ -1,3 +1,4 @@ +import { PdfCaptureError } from "./errors.js"; import { spawn } from "node:child_process"; import { existsSync } from "node:fs"; import { homedir } from "node:os"; @@ -26,7 +27,7 @@ async function readBoundedStream(stream: NodeJS.ReadableStream, maxBytes: number if (typeof value === "string") chunk = new TextEncoder().encode(value); else if (value instanceof Uint8Array) chunk = value; else throw new Error("PDF tool returned an unsupported output chunk"); - if (!bytes.append(chunk)) throw new Error(`PDF tool output exceeded ${maxBytes} bytes`); + if (!bytes.append(chunk)) throw new PdfCaptureError("limit-exceeded", `PDF tool output exceeded ${maxBytes} bytes`); } return new TextDecoder().decode(bytes.toUint8Array()); } @@ -37,7 +38,8 @@ export const runPdfToolCommand: PdfToolRunner = async ( ): Promise => { const executable = specification.command[0]; if (executable === undefined) throw new Error("PDF tool command is empty"); - const useProcessGroup = process.platform !== "win32"; + if (specification.processGroup !== undefined && !["new", "inherit"].includes(specification.processGroup)) throw new PdfCaptureError("invalid-input", "Invalid PDF process group policy"); + const useProcessGroup = process.platform !== "win32" && specification.processGroup !== "inherit"; const child = spawn(executable, specification.command.slice(1), { cwd: specification.cwd, detached: useProcessGroup, @@ -51,7 +53,7 @@ export const runPdfToolCommand: PdfToolRunner = async ( windowsHide: true, }); const exited = new Promise((resolveExit, rejectExit) => { - child.once("error", rejectExit); + child.once("error", (error: NodeJS.ErrnoException) => rejectExit(error.code === "ENOENT" ? new PdfCaptureError("tool-unavailable", "PDF tool executable is unavailable", { cause: error }) : error)); child.once("close", (code) => resolveExit(code ?? 1)); }); const signalProcessTree = (signal: NodeJS.Signals): void => { @@ -79,7 +81,7 @@ export const runPdfToolCommand: PdfToolRunner = async ( forceKillTimer = setTimeout(() => signalProcessTree("SIGKILL"), 1_000); }; const timer = setTimeout(() => { - requestStop(new Error(`PDF tool timed out after ${specification.timeoutMs}ms`)); + requestStop(new PdfCaptureError("timeout", `PDF tool timed out after ${specification.timeoutMs}ms`)); }, specification.timeoutMs); try { @@ -137,10 +139,10 @@ export function resolvePdfTools(dependencies: PdfCaptureDependencies = {}): PdfT ? discoverExecutable("tesseract", dependencies) : dependencies.tools.tesseract; if (pdfinfo === null) { - throw new Error("pdfinfo is required for PDF ingestion; install the Poppler command-line tools"); + throw new PdfCaptureError("tool-unavailable", "pdfinfo is required for PDF ingestion; install the Poppler command-line tools"); } if (pdftohtml === null) { - throw new Error("pdftohtml is required for PDF ingestion; install the Poppler command-line tools"); + throw new PdfCaptureError("tool-unavailable", "pdftohtml is required for PDF ingestion; install the Poppler command-line tools"); } return { pdfinfo, pdftohtml, tesseract }; }