diff --git a/README.md b/README.md index e3d5404..903ba74 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,23 @@ The initial extraction includes: It deliberately excludes Silent Guardian prose, characters, art, audio, secrets, and project-specific production routes. +## How PickBits Weaver works + +Weaver is opinionated about setup so that it can be trusted with a manuscript. +These are requirements, not suggestions: + +1. **Your book lives in its own folder, under Git.** Every change has a + history and can be undone. `weaver init` sets this up for you. +2. **Weaver is installed as a package, never copied into your book.** The AI + that works on your book cannot change the tool. +3. **You work with an AI that can run commands,** such as Claude Code. Chat + windows that can't run Weaver aren't supported. +4. **Your original files are never changed.** Import reads them; export writes + new files. + +`weaver doctor` checks all of this. Until it passes, Weaver won't write to your +book, and it tells you in plain words what to fix. + ## Quick start Requirements: Node.js 22 or newer. There are no runtime dependencies. @@ -30,14 +47,31 @@ npx weaver status --root ./my-book npx weaver grounding 1-1 --root ./my-book ``` +## Repair a finished book + +Use one Weaver project for each book in a series. Start an empty book, import +the finished Word, Markdown, text, or folder source, then check and export it. +The original source is read-only and is never changed, moved, or renamed. + +```bash +npx weaver init ./my-book --id my-book --title "My Book" --empty +npx weaver import ./finished-book.docx --root ./my-book +npx weaver check --root ./my-book +npx weaver export --format docx --out ./repaired-book.docx --root ./my-book +``` + +An imported book reports its narrative state as stale until a later workflow +bootstraps that derived state; this is expected, and its critical-path check +is skipped until a project-specific path is defined. + +Scene prose uses Markdown-style `*italics*`; literal asterisks and backslashes +are escaped. Bold Markdown is intentionally unsupported and is imported as +literal text. + The GitHub install works today. The equivalent registry package will be `@pickbitsai/weaver` after npm publication. -## Supported setup - -Keep the book in its own folder under Git. Install Weaver as a package outside -the book; never copy Weaver into the book folder. Weaver runs with an AI that -can run commands, such as Claude Code, and `weaver doctor` checks this setup. +## Keep narrative state current After writing or revising a scene, update its corresponding `narrative-state/-.md`, then explicitly accept the derived state: @@ -56,6 +90,8 @@ Re-derive them in order before accepting state again. | Command | Purpose | | --- | --- | | `init ` | Create a generic book workspace | +| `import ` | Import a finished DOCX, Markdown, text file, or folder into an empty book | +| `export --format md\|docx --out ` | Export the current manuscript without overwriting the source or an existing file | | `doctor` | Check the book folder, Git, host, and Weaver installation | | `status` | Summarize manuscript hash, words, scenes, and state freshness | | `check` | Run project, critical-path, repetition, and state gates | @@ -100,7 +136,3 @@ considered a release. Apache-2.0; see [LICENSE](LICENSE) and [NOTICE](NOTICE). Keep NOTICE with any copy or derivative. Versions up to and including 0.2.0 were released under the MIT License. - -## Historical License - -MIT © Mark Pickering and PickBits.AI diff --git a/contracts/import.schema.json b/contracts/import.schema.json new file mode 100644 index 0000000..61c0f80 --- /dev/null +++ b/contracts/import.schema.json @@ -0,0 +1,77 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/pickbitsai/weaver/main/contracts/import.schema.json", + "title": "Weaver book import record", + "type": "object", + "additionalProperties": false, + "required": ["$schema", "generator", "version", "imported_at", "source", "front_matter", "not_imported", "pov_rule", "chapters", "totals"], + "properties": { + "$schema": { "const": "https://pickbits.ai/schemas/weaver/import-1.json" }, + "generator": { "type": "string" }, + "version": { "type": "string" }, + "imported_at": { "type": "string", "format": "date-time" }, + "source": { + "type": "object", + "additionalProperties": false, + "required": ["file", "sha256", "kind"], + "properties": { + "file": { "type": "string", "minLength": 1 }, + "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "kind": { "enum": ["docx", "markdown", "text", "folder"] } + } + }, + "front_matter": { "type": "string" }, + "not_imported": { + "type": "object", + "additionalProperties": false, + "required": ["comments", "footnotes", "endnotes"], + "properties": { + "comments": { "type": "integer", "minimum": 0 }, + "footnotes": { "type": "integer", "minimum": 0 }, + "endnotes": { "type": "integer", "minimum": 0 } + } + }, + "pov_rule": { "enum": ["title", "none"] }, + "chapter_pattern": { "type": "string" }, + "chapters": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["number", "title", "pov", "scenes", "scene_count", "words"], + "properties": { + "number": { "type": "integer", "minimum": 1 }, + "title": { "type": "string" }, + "pov": { "type": ["string", "null"] }, + "scenes": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["number", "prose", "words", "sha256"], + "properties": { + "number": { "type": "integer", "minimum": 1 }, + "prose": { "type": "string" }, + "words": { "type": "integer", "minimum": 0 }, + "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" } + } + } + }, + "scene_count": { "type": "integer", "minimum": 1 }, + "words": { "type": "integer", "minimum": 0 } + } + } + }, + "totals": { + "type": "object", + "additionalProperties": false, + "required": ["chapters", "scenes", "words"], + "properties": { + "chapters": { "type": "integer", "minimum": 1 }, + "scenes": { "type": "integer", "minimum": 1 }, + "words": { "type": "integer", "minimum": 0 } + } + } + } +} diff --git a/contracts/project.schema.json b/contracts/project.schema.json index 84e2d28..63235fc 100644 --- a/contracts/project.schema.json +++ b/contracts/project.schema.json @@ -33,6 +33,21 @@ "chapters": { "type": "object", "additionalProperties": { "type": "string" } + }, + "source_book": { + "type": "object", + "additionalProperties": false, + "required": ["file", "sha256", "kind", "imported_at", "chapters", "scenes", "words", "pov_rule"], + "properties": { + "file": { "type": "string", "minLength": 1 }, + "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "kind": { "enum": ["docx", "markdown", "text", "folder"] }, + "imported_at": { "type": "string", "format": "date-time" }, + "chapters": { "type": "integer", "minimum": 1 }, + "scenes": { "type": "integer", "minimum": 1 }, + "words": { "type": "integer", "minimum": 0 }, + "pov_rule": { "enum": ["title", "none"] } + } } } } diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index 1c1e251..d42a7ad 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -8,6 +8,27 @@ Before working on a book, run: weaver doctor --root ./my-book ``` +## Repair path + +For a finished book, create a separate Weaver project with `--empty`; this +removes the starter scene and starter critical-path fixture while preserving +the generic project configuration. Importing never merges with existing scene +files and never writes to the source document. A series uses one project per +book. + +```bash +weaver init ./my-book --id my-book --title "My Book" --empty +weaver import ./finished-book.docx --root ./my-book +weaver status --root ./my-book +weaver check --root ./my-book +weaver export --format docx --out ./repaired-book.docx --root ./my-book +``` + +The import record under `imports/` preserves the source hash, front matter, +chapter and scene counts, POV decisions, and scene word counts. Imported books +begin with missing or stale narrative state by design; `check` reports that +state but does not fail on it until the project's state workflow is bootstrapped. + ## 1. Lock the architecture Before drafting, define the ending, scene-level beat map, reveal/reframe ledger, diff --git a/engine/export.mjs b/engine/export.mjs new file mode 100644 index 0000000..72b1da2 --- /dev/null +++ b/engine/export.mjs @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 Mark Pickering and PICKBITS LLC. Part of PickBits Weaver. +import { existsSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { deflateRawSync } from "node:zlib"; +import { loadScenes } from "./manuscript.mjs"; +import { GENERATOR } from "./identity.mjs"; +import { parseEscapedProse } from "./prose.mjs"; + +const DOS_TIME = 0; +const DOS_DATE = 33; +const CRC_TABLE = Array.from({ length: 256 }, (_, value) => { + let crc = value; + for (let bit = 0; bit < 8; bit += 1) crc = (crc >>> 1) ^ (crc & 1 ? 0xedb88320 : 0); + return crc >>> 0; +}); + +export function crc32(bytes) { + let crc = 0xffffffff; + for (const byte of bytes) crc = CRC_TABLE[(crc ^ byte) & 0xff] ^ (crc >>> 8); + return (crc ^ 0xffffffff) >>> 0; +} + +function u16(value) { + const result = Buffer.alloc(2); + result.writeUInt16LE(value, 0); + return result; +} + +function u32(value) { + const result = Buffer.alloc(4); + result.writeUInt32LE(value >>> 0, 0); + return result; +} + +export function zipEntries(entries, { method = "deflate", compression = null } = {}) { + const pairs = entries instanceof Map ? [...entries.entries()] : Array.isArray(entries) ? entries : Object.entries(entries); + method = compression || method; + const local = []; + const central = []; + let offset = 0; + for (const [name, input] of pairs) { + const filename = Buffer.from(name, "utf8"); + const content = Buffer.from(input); + if (method !== "deflate" && method !== "stored") throw new Error(`unsupported ZIP method: ${method}`); + const compressed = method === "stored" ? content : deflateRawSync(content, { level: 9 }); + const checksum = crc32(content); + const localHeader = Buffer.concat([ + u32(0x04034b50), u16(20), u16(0x0800), u16(method === "stored" ? 0 : 8), u16(DOS_TIME), u16(DOS_DATE), u32(checksum), + u32(compressed.length), u32(content.length), u16(filename.length), u16(0), filename, compressed + ]); + local.push(localHeader); + central.push(Buffer.concat([ + u32(0x02014b50), u16(20), u16(20), u16(0x0800), u16(method === "stored" ? 0 : 8), u16(DOS_TIME), u16(DOS_DATE), u32(checksum), + u32(compressed.length), u32(content.length), u16(filename.length), u16(0), u16(0), u16(0), u16(0), u32(0), u32(offset), filename + ])); + offset += localHeader.length; + } + const localBytes = Buffer.concat(local); + const centralBytes = Buffer.concat(central); + const end = Buffer.concat([ + u32(0x06054b50), u16(0), u16(0), u16(pairs.length), u16(pairs.length), u32(centralBytes.length), u32(localBytes.length), u16(0) + ]); + return Buffer.concat([localBytes, centralBytes, end]); +} + +export const writeZip = zipEntries; + +function xmlEscape(value) { + return String(value) + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +function textRun(text, italic = false) { + if (!text) return ""; + const properties = italic ? "" : ""; + const chunks = text.split(/([\t\n])/); + const body = chunks.map((chunk) => { + if (chunk === "\t") return ""; + if (chunk === "\n") return ""; + return chunk ? `${xmlEscape(chunk)}` : ""; + }).join(""); + return `${properties}${body}`; +} + +function markdownRuns(prose) { + return parseEscapedProse(prose).map((token) => textRun(token.text, token.italic)).join(""); +} + +function paragraph(text, style = "Normal", alignment = null, runs = null) { + const properties = [style ? `` : "", alignment ? `` : ""].join(""); + return `${properties}${runs || markdownRuns(text)}`; +} + +function chapterHeading(project, chapter) { + const title = project.chapters?.[String(chapter)]; + return `Chapter ${chapter}${title && title !== `Chapter ${chapter}` ? `: ${title}` : ""}`; +} + +function documentXml(project, scenes) { + const body = [paragraph(project.title, "Title", "center", textRun(project.title))]; + if (project.author) body.push(paragraph(`By ${project.author}`, "Normal", "center", textRun(`By ${project.author}`))); + let activeChapter = null; + for (const scene of scenes) { + if (scene.chapter !== activeChapter) { + activeChapter = scene.chapter; + const heading = chapterHeading(project, activeChapter); + body.push(paragraph(heading, "Heading1", "center", textRun(heading))); + } else { + body.push(paragraph("* * *", "Normal", "center", textRun("* * *"))); + } + for (const proseParagraph of scene.prose.split(/\n\s*\n/).filter(Boolean)) { + body.push(paragraph(proseParagraph.replace(/\n/g, "\n"))); + } + } + return `\n${body.join("")}`; +} + +function stylesXml() { + return `\n`; +} + +function contentTypesXml() { + return `\n`; +} + +function relationshipsXml() { + return `\n`; +} + +function documentRelationshipsXml() { + return `\n`; +} + +function coreXml(project) { + return `\n${xmlEscape(project.title)}${xmlEscape(project.author || "")}`; +} + +function appXml() { + return `\n${xmlEscape(GENERATOR)}`; +} + +export function buildDocx(project, scenes) { + const entries = [ + ["[Content_Types].xml", contentTypesXml()], + ["_rels/.rels", relationshipsXml()], + ["word/document.xml", documentXml(project, scenes)], + ["word/styles.xml", stylesXml()], + ["word/_rels/document.xml.rels", documentRelationshipsXml()], + ["docProps/core.xml", coreXml(project)], + ["docProps/app.xml", appXml()] + ]; + return zipEntries(entries); +} + +export function assembleExportMarkdown(project, scenes) { + const lines = [`# ${project.title}`, ""]; + let activeChapter = null; + for (const scene of scenes) { + if (scene.chapter !== activeChapter) { + activeChapter = scene.chapter; + lines.push(`## ${chapterHeading(project, activeChapter)}`, ""); + } else { + lines.push("* * *", ""); + } + lines.push(scene.prose, ""); + } + return `${lines.join("\n").trimEnd()}\n`; +} + +function recordedSourceTarget(root, project, outPath) { + if (!project.source_book?.file) return false; + return resolve(outPath).endsWith(`\\${project.source_book.file}`) || resolve(outPath).endsWith(`/${project.source_book.file}`); +} + +export function exportBook(root, project, format, outPath) { + if (!/^(md|docx)$/.test(format || "")) throw new Error("export format must be md or docx"); + if (!outPath) throw new Error("export requires --out "); + const destination = resolve(outPath); + if (existsSync(destination)) throw new Error(`refusing to overwrite export file: ${destination}`); + if (recordedSourceTarget(root, project, destination)) throw new Error("refusing to export onto the recorded source file name"); + const scenes = loadScenes(root, project); + const bytes = format === "docx" + ? buildDocx(project, scenes) + : Buffer.from(assembleExportMarkdown(project, scenes), "utf8"); + writeFileSync(destination, bytes); + return { format, out: destination, bytes: bytes.length }; +} diff --git a/engine/import.mjs b/engine/import.mjs new file mode 100644 index 0000000..e6034e5 --- /dev/null +++ b/engine/import.mjs @@ -0,0 +1,495 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 Mark Pickering and PICKBITS LLC. Part of PickBits Weaver. +import { createHash } from "node:crypto"; +import { inflateRawSync } from "node:zlib"; +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + statSync, + writeFileSync +} from "node:fs"; +import { basename, extname, join, resolve } from "node:path"; +import { countWords } from "./manuscript.mjs"; +import { GENERATOR, SCHEMA_BASE, VERSION } from "./identity.mjs"; +import { canonicalItalic, escapeLiteral, normaliseMarkdownProse } from "./prose.mjs"; + +export const DEFAULT_CHAPTER_PATTERN = "^(chapter|ch\\.)\\s+([0-9]+|[a-z-]+)\\b(.*)$"; + +const textDecoder = new TextDecoder("utf-8", { fatal: true }); +const BREAKS = new Set(["***", "* * *", "\\*\\*\\*", "\\* \\* \\*", "#", "~~~", "---", "§", "⁂"]); + +export function sourceSha256(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +function decodeUtf8(bytes, source) { + try { + return textDecoder.decode(bytes).replace(/\r\n?/g, "\n"); + } catch { + throw new Error(`${source}: invalid UTF-8 text source`); + } +} + +function naturalCompare(left, right) { + const a = left.toLocaleLowerCase().split(/(\d+)/).filter(Boolean); + const b = right.toLocaleLowerCase().split(/(\d+)/).filter(Boolean); + for (let index = 0; index < Math.max(a.length, b.length); index += 1) { + if (a[index] === undefined) return -1; + if (b[index] === undefined) return 1; + const aNumber = /^\d+$/.test(a[index]); + const bNumber = /^\d+$/.test(b[index]); + if (aNumber && bNumber && Number(a[index]) !== Number(b[index])) return Number(a[index]) - Number(b[index]); + if (a[index] !== b[index]) return a[index].localeCompare(b[index]); + } + return left.localeCompare(right); +} + +function xmlUnescape(value) { + return value.replace(/&(?:amp|lt|gt|quot|apos|#x[0-9a-f]+|#\d+);/gi, (entity) => { + if (entity === "&") return "&"; + if (entity === "<") return "<"; + if (entity === ">") return ">"; + if (entity === """) return '"'; + if (entity === "'") return "'"; + const code = entity[2].toLocaleLowerCase() === "x" + ? Number.parseInt(entity.slice(3, -1), 16) + : Number.parseInt(entity.slice(2, -1), 10); + return Number.isFinite(code) ? String.fromCodePoint(code) : entity; + }); +} + +function zipEntries(bytes) { + const endSignature = 0x06054b50; + let end = -1; + for (let index = bytes.length - 22; index >= 0; index -= 1) { + if (bytes.readUInt32LE(index) === endSignature) { + end = index; + break; + } + } + if (end < 0) throw new Error("source is not a valid ZIP archive"); + const count = bytes.readUInt16LE(end + 10); + const directorySize = bytes.readUInt32LE(end + 12); + const directoryOffset = bytes.readUInt32LE(end + 16); + if (directoryOffset + directorySize > bytes.length) throw new Error("source ZIP central directory is truncated"); + const entries = new Map(); + let cursor = directoryOffset; + for (let index = 0; index < count; index += 1) { + if (bytes.readUInt32LE(cursor) !== 0x02014b50) throw new Error("source ZIP has an invalid central directory"); + const method = bytes.readUInt16LE(cursor + 10); + const compressedSize = bytes.readUInt32LE(cursor + 20); + const uncompressedSize = bytes.readUInt32LE(cursor + 24); + const nameLength = bytes.readUInt16LE(cursor + 28); + const extraLength = bytes.readUInt16LE(cursor + 30); + const commentLength = bytes.readUInt16LE(cursor + 32); + const localOffset = bytes.readUInt32LE(cursor + 42); + const name = bytes.subarray(cursor + 46, cursor + 46 + nameLength).toString("utf8"); + if (localOffset + 30 > bytes.length || bytes.readUInt32LE(localOffset) !== 0x04034b50) { + throw new Error(`source ZIP entry has an invalid local header: ${name}`); + } + const localNameLength = bytes.readUInt16LE(localOffset + 26); + const localExtraLength = bytes.readUInt16LE(localOffset + 28); + const dataStart = localOffset + 30 + localNameLength + localExtraLength; + const dataEnd = dataStart + compressedSize; + if (dataEnd > bytes.length) throw new Error(`source ZIP entry is truncated: ${name}`); + const compressed = bytes.subarray(dataStart, dataEnd); + let content; + if (method === 0) content = Buffer.from(compressed); + else if (method === 8) content = inflateRawSync(compressed); + else throw new Error(`source DOCX uses unsupported ZIP compression: ${method}`); + if (content.length !== uncompressedSize) throw new Error(`source ZIP entry has an invalid size: ${name}`); + entries.set(name, content); + cursor += 46 + nameLength + extraLength + commentLength; + } + return entries; +} + +function xmlAttribute(tag, name) { + const match = new RegExp(`(?:^|\\s)(?:w:)?${name}=["']([^"']*)["']`).exec(tag); + return match ? xmlUnescape(match[1]) : null; +} + +function xmlTextFromRun(run) { + const italicTag = run.match(/]*)?\/>/i)?.[0]; + const italic = Boolean(italicTag) && !/\b(?:w:)?val=["'](?:0|false)["']/i.test(italicTag); + const tokens = []; + const tokenPattern = /]*)?>([\s\S]*?)<\/w:t>|]*)?\/>|]*)?\/>/gi; + let match; + while ((match = tokenPattern.exec(run))) { + if (match[1] !== undefined) tokens.push(xmlUnescape(match[1])); + else if (match[0].toLocaleLowerCase().startsWith(" segment.italic ? canonicalItalic(segment.value) : escapeLiteral(segment.value)).join(""); +} + +function countElements(xml, name, excludeTypes = []) { + if (!xml) return 0; + const pattern = new RegExp(`]*)>`, "gi"); + let count = 0; + let match; + while ((match = pattern.exec(xml))) { + const type = /(?:^|\s)w:type=["']([^"']+)["']/i.exec(match[1])?.[1]; + if (!excludeTypes.includes(type)) count += 1; + } + return count; +} + +function docxNotImported(entries) { + const comments = entries.get("word/comments.xml"); + const footnotes = entries.get("word/footnotes.xml"); + const endnotes = entries.get("word/endnotes.xml"); + return { + comments: countElements(comments ? decodeUtf8(comments, "word/comments.xml") : "", "comment"), + footnotes: countElements(footnotes ? decodeUtf8(footnotes, "word/footnotes.xml") : "", "footnote", ["separator", "continuationSeparator"]), + endnotes: countElements(endnotes ? decodeUtf8(endnotes, "word/endnotes.xml") : "", "endnote", ["separator", "continuationSeparator"]) + }; +} + +function docxParagraphs(xml) { + const paragraphs = []; + const paragraphPattern = /]*)?>[\s\S]*?<\/w:p>/gi; + let paragraphMatch; + while ((paragraphMatch = paragraphPattern.exec(xml))) { + const paragraph = paragraphMatch[0]; + const style = paragraph.match(/]*)?\/>/i)?.[0]; + const styleValue = style ? xmlAttribute(style, "val") : null; + const withoutDeletions = paragraph.replace(/]*)?>[\s\S]*?<\/w:del>/gi, ""); + const runs = []; + const runPattern = /]*)?>[\s\S]*?<\/w:r>/gi; + let runMatch; + while ((runMatch = runPattern.exec(withoutDeletions))) runs.push(xmlTextFromRun(runMatch[0])); + paragraphs.push({ text: docxRunText(runs), style: styleValue }); + } + return paragraphs; +} + +function readDocx(bytes, source) { + const entries = zipEntries(bytes); + const document = entries.get("word/document.xml"); + if (!document) throw new Error(`${source}: DOCX is missing word/document.xml`); + const xml = decodeUtf8(document, `${source} word/document.xml`); + if (/<(?:[A-Za-z_][\w.-]*:)?(?:ins|del|moveFrom|moveTo)\b/i.test(xml)) { + throw new Error("Your Word file still has tracked changes. Open it in Word, accept or reject all changes (Review → Accept / Reject), save, and import again."); + } + return { paragraphs: docxParagraphs(xml), notImported: docxNotImported(entries) }; +} + +function paragraphsFromText(text, kind) { + const paragraphs = []; + let lines = []; + const flush = () => { + if (lines.length) paragraphs.push({ text: lines.join("\n"), style: null }); + lines = []; + }; + for (const line of text.split("\n")) { + const trimmed = line.trim(); + if (!trimmed || (kind === "markdown" && /^#{1,2}\s+/.test(line)) || BREAKS.has(trimmed)) { + flush(); + if (trimmed) paragraphs.push({ text: line, style: null }); + } else { + lines.push(line); + } + } + flush(); + return paragraphs; +} + +function normaliseParagraph(value) { + return value.replace(/\r\n?/g, "\n").split("\n").map((line) => line.replace(/[ \t]+$/g, "")).join("\n"); +} + +function makeChapterPattern(pattern) { + try { + return new RegExp(pattern, "i"); + } catch (error) { + throw new Error(`invalid chapter pattern: ${error.message}`); + } +} + +function chapterMatch(text, pattern) { + const match = pattern.exec(text); + if (match) pattern.lastIndex = 0; + return match; +} + +function chapterTitle(text, match, markdown, pattern = new RegExp(DEFAULT_CHAPTER_PATTERN, "i")) { + if (markdown) { + const heading = text.replace(/^#{1,2}\s+/, "").trim(); + const headingMatch = chapterMatch(heading, pattern); + return headingMatch ? chapterTitle(heading, headingMatch, false, pattern) : heading; + } + if (!match) return text.trim(); + const suffix = match.length > 3 ? match[3] : match.length > 2 ? match[2] : ""; + return String(suffix || "").replace(/^\s*[:.\-–—]\s*/, "").trim(); +} + +function isBoundary(paragraph, pattern, kind) { + const text = paragraph.text.trim(); + return paragraph.style === "Heading1" || paragraph.style === "Title" || (kind === "markdown" && /^#{1,2}\s+/.test(text)) || Boolean(chapterMatch(text, pattern)); +} + +function isSceneBreak(text, kind, afterBoundary) { + const value = text.trim(); + if (!BREAKS.has(value)) return false; + return value !== "---" || kind !== "markdown" || afterBoundary; +} + +function finishChapter(chapter, povRule, previousPov) { + const title = chapter.title; + const pov = povRule === "none" ? null : title || previousPov || null; + const scenes = []; + let current = []; + const flush = () => { + const prose = current + .map(normaliseParagraph) + .map((paragraph) => chapter.kind === "markdown" ? normaliseMarkdownProse(paragraph) : chapter.kind === "text" ? escapeLiteral(paragraph) : paragraph) + .filter((paragraph) => paragraph.trim()) + .join("\n\n"); + if (prose) scenes.push({ prose, words: countWords(prose), sha256: sourceSha256(Buffer.from(prose, "utf8")) }); + current = []; + }; + for (const paragraph of chapter.paragraphs) { + if (isSceneBreak(paragraph.text, chapter.kind, true)) flush(); + else if (normaliseParagraph(paragraph.text).trim()) current.push(paragraph.text); + } + flush(); + return { + number: chapter.number, + title, + pov, + scenes, + scene_count: scenes.length, + words: scenes.reduce((sum, scene) => sum + scene.words, 0) + }; +} + +function parseDocument(paragraphs, { kind, sourceFile, pattern, povRule, forceChapter = false }) { + const boundaries = paragraphs.map((paragraph) => isBoundary(paragraph, pattern, kind)); + const hasBoundary = boundaries.some(Boolean); + const frontMatter = hasBoundary + ? paragraphs.slice(0, boundaries.indexOf(true)).map((paragraph) => normaliseParagraph(paragraph.text)).filter(Boolean).join("\n\n") + : ""; + const chapters = []; + let current = null; + let previousPov = null; + const begin = (paragraph, number) => { + const text = paragraph.text.trim(); + const markdown = kind === "markdown" && /^#{1,2}\s+/.test(text); + const match = chapterMatch(text, pattern); + current = { number, title: chapterTitle(text, match, markdown, pattern), paragraphs: [], kind }; + }; + paragraphs.forEach((paragraph, index) => { + if (boundaries[index]) { + if (current) { + const finished = finishChapter(current, povRule, previousPov); + if (finished.scenes.length) { + chapters.push(finished); + previousPov = finished.pov; + } + } + begin(paragraph, chapters.length + 1); + } else if (current) { + current.paragraphs.push(paragraph); + } + }); + if (current) { + const finished = finishChapter(current, povRule, previousPov); + if (finished.scenes.length) chapters.push(finished); + } + if (!hasBoundary && forceChapter) { + const fallback = finishChapter({ number: 1, title: "", paragraphs, kind }, povRule, previousPov); + return { front_matter: "", chapters: fallback.scenes.length ? [fallback] : [] }; + } + return { front_matter: frontMatter, chapters }; +} + +function readTextFile(path) { + const bytes = readFileSync(path); + const text = decodeUtf8(bytes, path); + const kind = extname(path).toLocaleLowerCase() === ".md" ? "markdown" : "text"; + return { bytes, text, kind }; +} + +export function readSource(sourcePath, { chapterPattern = DEFAULT_CHAPTER_PATTERN, povRule = "title" } = {}) { + if (!/^(title|none)$/.test(povRule)) throw new Error(`invalid POV rule: ${povRule}`); + const source = resolve(sourcePath); + if (!existsSync(source)) throw new Error(`source does not exist: ${source}`); + const pattern = makeChapterPattern(chapterPattern); + const sourceStat = statSync(source); + if (sourceStat.isDirectory()) { + const files = readdirSync(source, { withFileTypes: true }) + .filter((entry) => entry.isFile() && /\.(?:md|txt)$/i.test(entry.name)) + .map((entry) => entry.name) + .sort(naturalCompare); + const chapters = []; + const frontMatter = []; + const hash = createHash("sha256"); + for (const file of files) { + const path = join(source, file); + const { bytes, text, kind } = readTextFile(path); + hash.update(file).update("\0").update(bytes).update("\0"); + const parsed = parseDocument(paragraphsFromText(text, kind), { kind, sourceFile: file, pattern, povRule, forceChapter: true }); + if (parsed.front_matter) frontMatter.push(parsed.front_matter); + const chapter = parsed.chapters[0]; + if (chapter) { + chapter.source_file = file; + chapters.push(chapter); + } + } + let previousPov = null; + for (const chapter of chapters) { + chapter.pov = povRule === "none" ? null : chapter.title || previousPov || null; + previousPov = chapter.pov; + } + return { + kind: "folder", + sha256: hash.digest("hex"), + front_matter: frontMatter.join("\n\n"), + chapters, + not_imported: { comments: 0, footnotes: 0, endnotes: 0 } + }; + } + const bytes = readFileSync(source); + if (bytes.subarray(0, 4).equals(Buffer.from([0x50, 0x4b, 0x03, 0x04]))) { + const docx = readDocx(bytes, source); + const parsed = parseDocument(docx.paragraphs, { kind: "docx", sourceFile: basename(source), pattern, povRule, forceChapter: true }); + return { kind: "docx", sha256: sourceSha256(bytes), front_matter: parsed.front_matter, chapters: parsed.chapters, not_imported: docx.notImported }; + } + const text = decodeUtf8(bytes, source); + const kind = extname(source).toLocaleLowerCase() === ".md" ? "markdown" : "text"; + const parsed = parseDocument(paragraphsFromText(text, kind), { kind, sourceFile: basename(source), pattern, povRule, forceChapter: true }); + return { kind, sha256: sourceSha256(bytes), front_matter: parsed.front_matter, chapters: parsed.chapters, not_imported: { comments: 0, footnotes: 0, endnotes: 0 } }; +} + +function sourceState(sourcePath) { + const source = resolve(sourcePath); + const stat = statSync(source); + if (stat.isFile()) { + const bytes = readFileSync(source); + return { sha256: sourceSha256(bytes), mtimeMs: stat.mtimeMs, size: stat.size }; + } + const files = readdirSync(source, { withFileTypes: true }) + .filter((entry) => entry.isFile() && /\.(?:md|txt)$/i.test(entry.name)) + .map((entry) => entry.name) + .sort(naturalCompare); + const hash = createHash("sha256"); + const children = files.map((file) => { + const path = join(source, file); + const childStat = statSync(path); + const bytes = readFileSync(path); + hash.update(file).update("\0").update(bytes).update("\0"); + return { file, sha256: sourceSha256(bytes), mtimeMs: childStat.mtimeMs, size: childStat.size }; + }); + return { sha256: hash.digest("hex"), mtimeMs: stat.mtimeMs, children }; +} + +function assertSourceUnchanged(sourcePath, before) { + const after = sourceState(sourcePath); + if (JSON.stringify(before) !== JSON.stringify(after)) { + throw new Error(`source changed during import; refusing to write the imported book: ${resolve(sourcePath)}`); + } +} + +function hasSceneFiles(root, project) { + const manuscript = join(root, project.source_root); + if (!existsSync(manuscript)) return false; + return readdirSync(manuscript, { withFileTypes: true }).some((chapter) => { + if (!chapter.isDirectory() || !/^chapter-\d+$/.test(chapter.name)) return false; + return readdirSync(join(manuscript, chapter.name), { withFileTypes: true }).some((scene) => ( + scene.isDirectory() && /^scene-\d+$/.test(scene.name) && existsSync(join(manuscript, chapter.name, scene.name, "scene.md")) + )); + }); +} + +function sourceFileName(sourcePath) { + return basename(resolve(sourcePath)); +} + +function sceneFile(chapter, scene, sourceFile, importedAt) { + const pov = chapter.pov || ""; + return `**POV:** ${pov}\n**Source:** ${sourceFile}\n**Imported:** ${importedAt}\n**Status:** Imported\n\n---\n\n${scene.prose}\n\n---\n\n**Words:** ${scene.words}\n`; +} + +export function importBook(root, project, sourcePath, { chapterPattern = DEFAULT_CHAPTER_PATTERN, povRule = "title", importedAt = new Date().toISOString() } = {}) { + if (hasSceneFiles(root, project)) { + throw new Error("refusing import: this book already contains scene files; import only into an empty book"); + } + const before = sourceState(sourcePath); + const source = readSource(sourcePath, { chapterPattern, povRule }); + assertSourceUnchanged(sourcePath, before); + if (!source.chapters.length) throw new Error("source contains no chapters with prose"); + const file = sourceFileName(sourcePath); + const chapters = source.chapters.map((chapter, index) => ({ ...chapter, number: index + 1, source_file: chapter.source_file || file })); + const totals = { + chapters: chapters.length, + scenes: chapters.reduce((sum, chapter) => sum + chapter.scenes.length, 0), + words: chapters.reduce((sum, chapter) => sum + chapter.words, 0) + }; + const record = { + $schema: `${SCHEMA_BASE}import-1.json`, + generator: GENERATOR, + version: VERSION, + imported_at: importedAt, + source: { file, sha256: source.sha256, kind: source.kind }, + front_matter: source.front_matter, + not_imported: source.not_imported || { comments: 0, footnotes: 0, endnotes: 0 }, + pov_rule: povRule, + chapter_pattern: chapterPattern, + chapters: chapters.map((chapter) => ({ + number: chapter.number, + title: chapter.title, + pov: chapter.pov, + scenes: chapter.scenes.map((scene, sceneIndex) => ({ number: sceneIndex + 1, prose: scene.prose, words: scene.words, sha256: scene.sha256 })), + scene_count: chapter.scenes.length, + words: chapter.words + })), + totals + }; + if (!totals.scenes) throw new Error("source contains no prose scenes"); + const manuscript = join(root, project.source_root); + const chapterWidth = Math.max(2, String(chapters.length).length); + for (const chapter of chapters) { + const sceneWidth = Math.max(2, String(chapter.scenes.length).length); + for (const [sceneIndex, scene] of chapter.scenes.entries()) { + const path = join(manuscript, `chapter-${String(chapter.number).padStart(chapterWidth, "0")}`, `scene-${String(sceneIndex + 1).padStart(sceneWidth, "0")}`, "scene.md"); + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, sceneFile(chapter, scene, chapter.source_file, importedAt)); + } + } + const nextProject = { + ...project, + chapters: Object.fromEntries(chapters.map((chapter) => [String(chapter.number), chapter.title || `Chapter ${chapter.number}`])), + source_book: { + file, + sha256: source.sha256, + kind: source.kind, + imported_at: importedAt, + chapters: totals.chapters, + scenes: totals.scenes, + words: totals.words, + pov_rule: povRule + } + }; + writeFileSync(join(root, "project.json"), `${JSON.stringify(nextProject, null, 2)}\n`); + const imports = join(root, "imports"); + mkdirSync(imports, { recursive: true }); + const importPath = join(imports, `${importedAt.slice(0, 10)}-${source.sha256.slice(0, 12)}.json`); + writeFileSync(importPath, `${JSON.stringify(record, null, 2)}\n`); + assertSourceUnchanged(sourcePath, before); + return { record, importPath, project: nextProject }; +} diff --git a/engine/init.mjs b/engine/init.mjs index 38f3d6b..81ff2c9 100644 --- a/engine/init.mjs +++ b/engine/init.mjs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright 2026 Mark Pickering and PICKBITS LLC. Part of PickBits Weaver. -import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { spawnSync } from "node:child_process"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; @@ -9,7 +9,7 @@ import { PROJECT_ID_PATTERN } from "./project.mjs"; const ENGINE_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const TEMPLATE_ROOT = join(ENGINE_ROOT, "templates", "book"); -export function initializeProject(target, { projectId = "untitled-book", title = "Untitled Book" } = {}) { +export function initializeProject(target, { projectId = "untitled-book", title = "Untitled Book", empty = false } = {}) { if (!PROJECT_ID_PATTERN.test(projectId)) { throw new Error("project id must use lowercase letters, numbers, and hyphens"); } @@ -20,6 +20,12 @@ export function initializeProject(target, { projectId = "untitled-book", title = } mkdirSync(destination, { recursive: true }); cpSync(TEMPLATE_ROOT, destination, { recursive: true }); + if (empty) { + rmSync(join(destination, "manuscript", "chapter-01"), { recursive: true, force: true }); + rmSync(join(destination, "narrative-state", "1-1.md"), { force: true }); + rmSync(join(destination, "outline", "scenes", "1-1.md"), { force: true }); + rmSync(join(destination, "quality", "critical-path.json"), { force: true }); + } const projectPath = join(destination, "project.json"); const project = JSON.parse(readFileSync(projectPath, "utf8")); project.project_id = projectId; diff --git a/engine/manuscript.mjs b/engine/manuscript.mjs index 5ce4cf2..4433ed3 100644 --- a/engine/manuscript.mjs +++ b/engine/manuscript.mjs @@ -16,7 +16,7 @@ export function extractProse(text, source = "scene") { } export function countWords(text) { - return text.match(/[\p{L}\p{N}]+(?:[’'][\p{L}\p{N}]+)*/gu)?.length || 0; + return text.match(/[\p{L}\p{N}]+(?:['\u2019][\p{L}\p{N}]+)*/gu)?.length || 0; } export function loadScenes(root, project) { diff --git a/engine/prose.mjs b/engine/prose.mjs new file mode 100644 index 0000000..776bc79 --- /dev/null +++ b/engine/prose.mjs @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 Mark Pickering and PICKBITS LLC. Part of PickBits Weaver. + +export function escapeLiteral(value) { + return String(value).replaceAll("\\", "\\\\").replaceAll("*", "\\*"); +} + +function decodeMarkdownEscape(value, index) { + const next = value[index + 1]; + if (next === "*" || next === "_" || next === "\\") return { value: next, nextIndex: index + 2 }; + return { value: "\\", nextIndex: index + 1 }; +} + +function pushToken(tokens, italic, text) { + if (!text) return; + const last = tokens.at(-1); + if (last?.italic === italic) last.text += text; + else tokens.push({ italic, text }); +} + +function pushItalic(tokens, text) { + const leading = text.match(/^[\t \r\n]*/u)[0]; + const trailing = text.match(/[\t \r\n]*$/u)[0]; + const coreEnd = text.length - trailing.length; + const core = text.slice(leading.length, coreEnd); + if (!core) { + pushToken(tokens, false, text); + return; + } + pushToken(tokens, false, leading); + pushToken(tokens, true, core); + pushToken(tokens, false, trailing); +} + +function closingMarker(value, start, marker) { + if (/[\t \r\n]/u.test(value[start] || "")) return -1; + for (let index = start; index < value.length; index += 1) { + if (value[index] === "\\") { + index += value[index + 1] === "*" || value[index + 1] === "_" || value[index + 1] === "\\" ? 1 : 0; + continue; + } + if (value[index] !== marker || value[index - 1] === marker || value[index + 1] === marker) continue; + const inner = value.slice(start, index); + if (inner && !/[\r\n\t ]/u.test(inner.at(-1)) && !/[\r\n]/u.test(inner)) return index; + } + return -1; +} + +function normaliseMarkdownTokens(value) { + const tokens = []; + for (let index = 0; index < value.length;) { + const character = value[index]; + if (character === "\\") { + const decoded = decodeMarkdownEscape(value, index); + pushToken(tokens, false, decoded.value); + index = decoded.nextIndex; + continue; + } + if (character === "*" || character === "_") { + if (value[index + 1] === character) { + pushToken(tokens, false, value.slice(index, index + 2)); + index += 2; + continue; + } + const end = value[index - 1] === character || /[\t \r\n]/u.test(value[index + 1] || "") + ? -1 + : closingMarker(value, index + 1, character); + if (end >= 0) { + let inner = ""; + for (let innerIndex = index + 1; innerIndex < end;) { + if (value[innerIndex] === "\\") { + const decoded = decodeMarkdownEscape(value, innerIndex); + inner += decoded.value; + innerIndex = decoded.nextIndex; + } else { + inner += value[innerIndex]; + innerIndex += 1; + } + } + pushItalic(tokens, inner); + index = end + 1; + continue; + } + } + pushToken(tokens, false, character); + index += 1; + } + return tokens; +} + +export function normaliseMarkdownProse(value) { + return normaliseMarkdownTokens(String(value)) + .map((token) => token.italic ? `*${escapeLiteral(token.text)}*` : escapeLiteral(token.text)) + .join(""); +} + +function escapedProseTokens(value) { + const tokens = []; + for (let index = 0; index < value.length;) { + const character = value[index]; + if (character === "\\") { + const next = value[index + 1]; + if (next === "*" || next === "\\") { + pushToken(tokens, false, next); + index += 2; + } else { + pushToken(tokens, false, "\\"); + index += 1; + } + continue; + } + if (character === "*" && value[index + 1] !== "*") { + const end = value[index - 1] === "*" || /[\t \r\n]/u.test(value[index + 1] || "") + ? -1 + : closingMarker(value, index + 1, "*"); + if (end >= 0) { + let inner = ""; + for (let innerIndex = index + 1; innerIndex < end;) { + if (value[innerIndex] === "\\" && (value[innerIndex + 1] === "*" || value[innerIndex + 1] === "\\")) { + inner += value[innerIndex + 1]; + innerIndex += 2; + } else { + inner += value[innerIndex]; + innerIndex += 1; + } + } + pushItalic(tokens, inner); + index = end + 1; + continue; + } + } + pushToken(tokens, false, character); + index += 1; + } + return tokens; +} + +export function parseEscapedProse(value) { + return escapedProseTokens(String(value)); +} + +export function canonicalItalic(value) { + const tokens = []; + pushItalic(tokens, String(value)); + return tokens.map((token) => token.italic ? `*${escapeLiteral(token.text)}*` : escapeLiteral(token.text)).join(""); +} diff --git a/engine/release.mjs b/engine/release.mjs index 37a504b..acb8901 100644 --- a/engine/release.mjs +++ b/engine/release.mjs @@ -22,7 +22,9 @@ const escapeHtml = (value) => value export function runQualityChecks(root, project) { const scenes = loadScenes(root, project); - const criticalPath = validateCriticalPath(root, scenes); + const criticalPath = project.source_book && !existsSync(join(root, "quality", "critical-path.json")) + ? { ok: true, skipped: true, total: 0, intact: 0, missing: [] } + : validateCriticalPath(root, scenes); const duplicates = findHighSeverityDuplicates(scenes); const unapprovedDuplicates = filterIntentionalEchoes(root, duplicates); const state = narrativeStateStatus(root, project); diff --git a/integrity.json b/integrity.json index bcb6132..d0029ab 100644 --- a/integrity.json +++ b/integrity.json @@ -5,11 +5,12 @@ "files": { "LICENSE": "3954ad2ee7d56cfda60326c71dcfd3d7edd79a44727f51b5ca5e4b7c33b8c6d6", "NOTICE": "83abe7f22edf5a06f7b4814e8b4b4b124e1b612bd81dd4c6542c0b119fab4976", - "README.md": "ec8605dd0bca0080675e34bf4bb48fe79e7b507938a67e397984a7bb39fdac1b", + "README.md": "446428a5447426c0731027f61791ce8dfc5beabe707aa7b9ee07577822d81844", "SECURITY.md": "5c9c72ffaa8dfdc744360ea666d2082c68bba2479c61d13810bbd595498afe2b", + "contracts/import.schema.json": "68a9be97050a3e58bacbc984c9e1050d2b0b1db72066af96e0a5fb4019fe9e85", "contracts/integrity.schema.json": "1a8cb4b28748b0f2bfeffc4d0210f5f422d1b69cfed3a386ff60c5de0bdd8d08", "contracts/loop-manifest.schema.json": "edb1022ab2434ea24890fb0a14ad7daee7dda190968c60e9ab306250412e6258", - "contracts/project.schema.json": "4f864f971de472e54988ca0df6323f168a07e37a1885b16b1c7b3f4e88b4c456", + "contracts/project.schema.json": "b6cf551a2f688898c941ea098ee1d0b18d30bcb008a01cb2803fa43f3e5ceda9", "contracts/provenance.schema.json": "bea26ddbcbfef8c046b0707e9a2f6dff8913cd9531b6267d3f374f821919d107", "contracts/release-manifest.schema.json": "65175761fc3095419ceb01b28457e6615c57c6eff4baceb2867128da73303db4", "contracts/state-manifest.schema.json": "e6c0c2ce39c81f5db5feed31f7e7432a694eeb04ab329445dc1fbac29aff553f", @@ -17,14 +18,17 @@ "docs/HANDOFF.md": "6e37433f60c54989abb1171c6989ed9c0cbab83be0d414d3d36ff6e985a9ee8c", "docs/LOOP.md": "bf3098c9afbcac6254d1682e87ad6b1e6ed798f0cb5e22a5f76a8fd3941fe8d5", "docs/ORIGIN.md": "d66e5d9a0c3e2c05a983c790dd616fc051d2666417ae0e1fd697478e5ff4cda8", - "docs/WORKFLOW.md": "0e7ba317ac72c395fc1ccdbfa6cf62ff5b8f7b27d92fb8a4b68229935e728b5c", + "docs/WORKFLOW.md": "7688db4a2c68b559ef92a61a3f4d20720949cab3db492ac3561fb04bc6f3c1be", "engine/doctor.mjs": "1e029539b16339f660347653d93ed1c003172f31c0d859a1f9edd0a4dd22ba5a", + "engine/export.mjs": "092ad9a4687cf24ca5daa822c844879c8314adc227542cf46ec7b871e43f9f3c", "engine/identity.mjs": "670f211b093aa783a0d6ed24be566239c438deb9da75e9ef20748e2c1b057364", - "engine/init.mjs": "0eca469117de0e406a4f94b78c6b5a67f3dfe1d6eaedc0f9512d154c5a1dd58a", + "engine/import.mjs": "36b5f8475563d4fd0ea0ca9209d0232598e53d824733137cf46b8dbc2d3a81cc", + "engine/init.mjs": "a09729d6f34049b9629b9bba152bf1a7d190158d47c4e1e73b3d01ab66235475", "engine/integrity.mjs": "5c9ae6d22bbd28515887678e0e1ac214edd77ef2d2312fab63eda52407301e9f", - "engine/manuscript.mjs": "6c25aa43f1939d4779127a5782579e265ea0aa1642a092ac9f30ad611a84d0be", + "engine/manuscript.mjs": "eb03ce6d453b9b6d3a7e44b5d1a8d40eb88dcb86ae3963aaf0b4597345b14a59", "engine/project.mjs": "9f31f86a6cee011ea2486b7773491e5f6529a0b7f28e14094f1fb436c6ab2b81", - "engine/release.mjs": "8cf1bd709d73481143745499daaf3d3e1e1fbfdcfc976ce2e6955ccea6db65ab", + "engine/prose.mjs": "551c5c2e263c67a72d6a1f7f90876e36f0a0d09c3f84f184be680d1e9cb53bcc", + "engine/release.mjs": "08724cb016ea874e2b01b2079138744359c08d5d338fb8e699a0af61527b2c80", "engine/state.mjs": "89204ffd1d349761d8977860f976099d784cd7b14be0ab91f63b3d9a61bcccb1", "loop.manifest.json": "340b33bb8b03c60ce4997b9182baa18435fc3d6716d660ea8d15ae466663abc0", "package.json": "d1b143a5559d860707d5085265a175e64ef07af09cf12411634e3522c0cb39be", @@ -34,7 +38,7 @@ "prompts/reviewers/style.md": "4479a4cb676627ca1ff4f7116f15e0cdc1f50fe2763db1388371d80570c9cdd4", "prompts/state-extractor.md": "b2d92d014892f2a1c201003b1cd16cf9d0adcc5e8611f7d350eab612b8947cc5", "scripts/integrity.mjs": "78c0bcdf7b5858de517f02eb09db1febf6d33eec787f09e7c24178f0cb263243", - "scripts/weaver.mjs": "2b780f8bc9f8bd452231ec31a049d871cc14204f8834417967356badaa5f2678", + "scripts/weaver.mjs": "927681518994c33598668fa9fc740df731cde14ae63574c4b22550d1bc668e7c", "templates/book/AGENTS.md": "6bb83885ca4498cfd5e994f2b36cd1688db04d6174905b0a813c5fbedd7c4a10", "templates/book/README.md": "c33756ab69072410e4583f44daf21d08fa6c576816c5ed240284850d6e4b2659", "templates/book/manuscript/chapter-01/scene-01/scene.md": "b17ea1077a47bcca702be9646c5aa3fbf6f557625e1daedfed30a7dfd684f5c6", diff --git a/scripts/weaver.mjs b/scripts/weaver.mjs index 71b5a67..aadc4b6 100644 --- a/scripts/weaver.mjs +++ b/scripts/weaver.mjs @@ -8,6 +8,8 @@ import { initializeProject } from "../engine/init.mjs"; import { loadScenes, manuscriptHash } from "../engine/manuscript.mjs"; import { readProject } from "../engine/project.mjs"; import { buildRelease, runQualityChecks } from "../engine/release.mjs"; +import { DEFAULT_CHAPTER_PATTERN, importBook } from "../engine/import.mjs"; +import { exportBook } from "../engine/export.mjs"; import { acceptNarrativeState, buildGroundingPacket, narrativeStateStatus } from "../engine/state.mjs"; import { GENERATOR } from "../engine/identity.mjs"; @@ -39,6 +41,13 @@ function print(value) { process.stdout.write(`${typeof value === "string" ? value : JSON.stringify(value, null, 2)}\n`); } +function importOmissionsSummary(notImported = {}) { + const comments = Number(notImported.comments || 0); + const footnotes = Number(notImported.footnotes || 0) + Number(notImported.endnotes || 0); + const plural = (count, singular) => `${count} ${singular}${count === 1 ? "" : "s"}`; + return `Not imported: ${plural(comments, "comment")}, ${plural(footnotes, "footnote")}.`; +} + function doctorReport(root) { return runDoctor({ bookRoot: root }); } @@ -89,7 +98,8 @@ try { if (!target) throw new Error("usage: weaver init [--id book-id] [--title \"Book Title\"]"); const destination = initializeProject(target, { projectId: option("--id", "untitled-book"), - title: option("--title", "Untitled Book") + title: option("--title", "Untitled Book"), + empty: argv.includes("--empty") }); print(destination); const gitCheck = doctorReport(destination).checks.find((check) => check.id === "git-available"); @@ -163,6 +173,29 @@ try { if (!releaseId) throw new Error("usage: weaver release --id beta-01 [--root directory] [--force]"); print(buildRelease(root, project, releaseId, { force: argv.includes("--force") })); } + } else if (command === "import") { + const source = positional(); + if (!source) throw new Error("usage: weaver import [--root directory] [--pov title|none] [--chapter-pattern regex]"); + if (refuseIfDoctorBlocks(root, command)) process.exitCode = 1; + else { + const { project } = projectContext(); + const result = importBook(root, project, source, { + povRule: option("--pov", "title"), + chapterPattern: option("--chapter-pattern", DEFAULT_CHAPTER_PATTERN) + }); + if (argv.includes("--json")) print(result.record); + else { + print(`Imported ${result.record.totals.chapters} chapter(s), ${result.record.totals.scenes} scene(s), ${result.record.totals.words} word(s).`); + print(importOmissionsSummary(result.record.not_imported)); + for (const chapter of result.record.chapters) print(`Chapter ${chapter.number}: ${chapter.title || "(untitled)"} — POV ${chapter.pov || "(none)"}`); + } + } + } else if (command === "export") { + if (refuseIfDoctorBlocks(root, command)) process.exitCode = 1; + else { + const { project } = projectContext(); + print(exportBook(root, project, option("--format"), option("--out"))); + } } else if (command === "version" || command === "--version") { warnAboutDoctor(root); print(GENERATOR); @@ -171,28 +204,33 @@ try { print(`${GENERATOR}: provenance-aware book pipeline Usage: - weaver init [--id book-id] [--title "Book Title"] + weaver init [--id book-id] [--title "Book Title"] [--empty] weaver doctor [--root directory] [--json] weaver status [--root directory] weaver check [--root directory] weaver state:status [--root directory] weaver state:accept [--root directory] [--through scene-id] weaver grounding [--root directory] [--output file] - weaver release --id beta-01 [--root directory] [--force]`); + weaver release --id beta-01 [--root directory] [--force] + weaver import [--root directory] [--pov title|none] [--chapter-pattern regex] [--json] + weaver export --format md|docx --out [--root directory]`); } else { print(`${GENERATOR}: provenance-aware book pipeline Usage: - weaver init [--id book-id] [--title "Book Title"] + weaver init [--id book-id] [--title "Book Title"] [--empty] weaver doctor [--root directory] [--json] weaver status [--root directory] weaver check [--root directory] weaver state:status [--root directory] weaver state:accept [--root directory] [--through scene-id] weaver grounding [--root directory] [--output file] - weaver release --id beta-01 [--root directory] [--force]`); + weaver release --id beta-01 [--root directory] [--force] + weaver import [--root directory] [--pov title|none] [--chapter-pattern regex] [--json] + weaver export --format md|docx --out [--root directory]`); } } catch (error) { - process.stderr.write(`${error.stack || error.message}\n`); + // Authors read this output: say what went wrong in plain words, never a stack trace. + process.stderr.write(`${error.message || error}\n`); process.exitCode = 1; } diff --git a/test/consumer-install.mjs b/test/consumer-install.mjs index c08e323..684f68a 100644 --- a/test/consumer-install.mjs +++ b/test/consumer-install.mjs @@ -11,6 +11,8 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const sandbox = mkdtempSync(join(tmpdir(), "weaver-consumer-")); const consumer = join(sandbox, "consumer"); const book = join(consumer, "book"); +const repairBook = join(consumer, "repair-book"); +const finishedSource = join(consumer, "finished.md"); const npmCli = process.env.npm_execpath; const env = { ...process.env, NPM_CONFIG_CACHE: join(sandbox, "npm-cache") }; assert.ok(npmCli, "npm_execpath is required; run this check through npm"); @@ -70,6 +72,22 @@ try { assert.match(readFileSync(installedNotice, "utf8"), /PickBits Weaver/); weaver(["init", book, "--id", "consumer-book", "--title", "Consumer Book"]); + weaver(["init", repairBook, "--id", "consumer-repair", "--title", "Consumer Repair", "--empty"]); + writeFileSync(finishedSource, "# Wren\n\nA finished scene with an em — dash.\n\n* * *\n\nA second scene."); + weaver(["import", finishedSource, "--root", repairBook]); + // An imported book has no accepted narrative state yet: check must say so, and release must + // refuse until state is current. Export (the repair path) does not depend on state. + const repairCheckResult = weaverResult(["check", "--root", repairBook]); + assert.equal(repairCheckResult.status, 1, repairCheckResult.stderr); + const repairCheck = JSON.parse(repairCheckResult.stdout); + assert.deepEqual(repairCheck.issues, [`narrative state is stale from scene ${repairCheck.narrative_state.first_stale}`]); + const repairRelease = weaverResult(["release", "--root", repairBook, "--id", "too-early"]); + assert.notEqual(repairRelease.status, 0); + assert.equal(existsSync(join(repairBook, "releases", "too-early")), false); + const repairDocx = join(consumer, "consumer-repair.docx"); + weaver(["export", "--format", "docx", "--out", repairDocx, "--root", repairBook]); + assert.equal(readFileSync(repairDocx).subarray(0, 2).toString(), "PK"); + const doctor = JSON.parse(weaver(["doctor", "--json", "--root", book])); assert.equal(doctor.checks.find((check) => check.id === "install-integrity").status, "pass"); assert.equal(doctor.ok, true); diff --git a/test/repair.test.mjs b/test/repair.test.mjs new file mode 100644 index 0000000..516d454 --- /dev/null +++ b/test/repair.test.mjs @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 Mark Pickering and PICKBITS LLC. Part of PickBits Weaver. +import assert from "node:assert/strict"; +import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { inflateRawSync } from "node:zlib"; +import test from "node:test"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { exportBook, zipEntries } from "../engine/export.mjs"; +import { importBook, readSource } from "../engine/import.mjs"; +import { initializeProject } from "../engine/init.mjs"; +import { countWords, loadScenes } from "../engine/manuscript.mjs"; +import { readProject } from "../engine/project.mjs"; + +const ROOT = resolve(fileURLToPath(new URL("..", import.meta.url))); + +const DOCUMENT_XML = ` + +Chapter 1: Wren +Curly “quotes” & em — dash.Tab +Italic words stay. +Keep accepted text. +* * * +Second scene. +Chapter 4 +Untitled chapter prose. +`; + +const TRACKED_DOCUMENT_XML = ` + +Chapter 1 +inserteddeleted +`; + +const COMMENTS_XML = ``; +const FOOTNOTES_XML = ``; +const ENDNOTES_XML = ``; + +function gitInit(root) { + const result = spawnSync("git", ["init", "-q"], { cwd: root, encoding: "utf8" }); + assert.equal(result.status, 0, result.stderr); +} + +function book(empty = true) { + const root = mkdtempSync(join(tmpdir(), "weaver-repair-")); + initializeProject(root, { projectId: "repair-book", title: "Repair Book", empty }); + gitInit(root); + return root; +} + +function fixture(root, method = "deflate", xml = DOCUMENT_XML, name = "finished") { + const path = join(root, `${name}-${method}.docx`); + writeFileSync(path, zipEntries([["word/document.xml", xml]], { method })); + return path; +} + +function zipText(bytes, wanted) { + let offset = 0; + while (bytes.readUInt32LE(offset) === 0x04034b50) { + const method = bytes.readUInt16LE(offset + 8); + const compressedSize = bytes.readUInt32LE(offset + 18); + const nameLength = bytes.readUInt16LE(offset + 26); + const extraLength = bytes.readUInt16LE(offset + 28); + const name = bytes.subarray(offset + 30, offset + 30 + nameLength).toString("utf8"); + const start = offset + 30 + nameLength + extraLength; + const content = bytes.subarray(start, start + compressedSize); + if (name === wanted) return (method === 0 ? content : inflateRawSync(content)).toString("utf8"); + offset = start + compressedSize; + } + throw new Error(`missing ZIP entry: ${wanted}`); +} + +test("DOCX import reads deflate and stored ZIPs, preserves prose, and records POV carry-forward", () => { + const root = book(); + try { + for (const method of ["deflate", "stored"]) { + const source = fixture(root, method); + const parsed = readSource(source); + assert.equal(parsed.kind, "docx"); + assert.deepEqual(parsed.chapters.map((chapter) => ({ title: chapter.title, pov: chapter.pov, scenes: chapter.scene_count })), [ + { title: "Wren", pov: "Wren", scenes: 2 }, + { title: "", pov: "Wren", scenes: 1 } + ]); + assert.match(parsed.chapters[0].scenes[0].prose, /em — dash/); + assert.match(parsed.chapters[0].scenes[0].prose, /\*Italic words\*/); + assert.match(parsed.chapters[0].scenes[0].prose, /Keep accepted text/); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("literal asterisks survive Markdown to DOCX and back without false italics", () => { + const root = book(); + const second = book(); + const source = join(root, "source.md"); + try { + writeFileSync(source, "# Chapter 1\n\nHe said, \"Get the f*** out,\" and 5 * 3 made fifteen.\n"); + importBook(root, readProject(root), source, { importedAt: "2026-09-25T00:00:00.000Z" }); + const expected = 'He said, "Get the f\\*\\*\\* out," and 5 \\* 3 made fifteen.'; + assert.equal(loadScenes(root, readProject(root))[0].prose, expected); + const exported = join(root, "round-trip.docx"); + exportBook(root, readProject(root), "docx", exported); + const xml = zipText(readFileSync(exported), "word/document.xml"); + assert.match(xml, /f\*\*\* out/); + assert.match(xml, /5 \* 3/); + assert.doesNotMatch(xml, /f\*\*/); + importBook(second, readProject(second), exported, { importedAt: "2026-09-25T00:00:00.000Z" }); + assert.equal(loadScenes(second, readProject(second))[0].prose, expected); + } finally { + rmSync(root, { recursive: true, force: true }); + rmSync(second, { recursive: true, force: true }); + } +}); + +test("scene breaks are literal centered paragraphs and titles use Title style", () => { + const root = book(); + const source = join(root, "source.md"); + try { + writeFileSync(source, "# Chapter 1\n\nOne scene.\n\n* * *\n\nTwo scene.\n"); + importBook(root, readProject(root), source); + const exported = join(root, "scene-break.docx"); + exportBook(root, readProject(root), "docx", exported); + const xml = zipText(readFileSync(exported), "word/document.xml"); + assert.match(xml, /<\/w:pPr>Repair Book<\/w:t><\/w:r><\/w:p>/); + assert.match(xml, /<\/w:pPr>\* \* \*<\/w:t><\/w:r><\/w:p>/); + assert.doesNotMatch(xml, /\* \* \*<\/w:t><\/w:r> { + const textRoot = book(); + const docxRoot = book(); + try { + const text = join(textRoot, "stars.txt"); + writeFileSync(text, "Chapter 1\n\n**stars** and \\slash.\n"); + importBook(textRoot, readProject(textRoot), text); + assert.equal(loadScenes(textRoot, readProject(textRoot))[0].prose, "\\*\\*stars\\*\\* and \\\\slash."); + const mixedXml = `Chapter 1Not yet, she saw f*** and 5 * 3.`; + const source = fixture(docxRoot, "deflate", mixedXml, "mixed"); + const parsed = readSource(source); + assert.equal(parsed.chapters[0].scenes[0].prose, "*Not yet,* she saw f\\*\\*\\* and 5 \\* 3."); + const expected = parsed.chapters[0].scenes[0].prose; + importBook(docxRoot, readProject(docxRoot), source); + const exported = join(docxRoot, "mixed-export.docx"); + exportBook(docxRoot, readProject(docxRoot), "docx", exported); + assert.equal(readSource(exported).chapters[0].scenes[0].prose, expected); + } finally { + rmSync(textRoot, { recursive: true, force: true }); + rmSync(docxRoot, { recursive: true, force: true }); + } +}); + +test("tracked changes refuse import and leave the book untouched", () => { + const root = book(); + try { + const source = fixture(root, "deflate", TRACKED_DOCUMENT_XML, "tracked"); + assert.throws(() => importBook(root, readProject(root), source), /Your Word file still has tracked changes\. Open it in Word, accept or reject all changes \(Review → Accept \/ Reject\), save, and import again\./); + assert.equal(existsSync(join(root, "imports")), false); + assert.equal(existsSync(join(root, "manuscript", "chapter-01")), false); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("import records and reports comments, footnotes, and endnotes left out", () => { + const root = book(); + try { + const source = join(root, "annotated.docx"); + writeFileSync(source, zipEntries([ + ["word/document.xml", DOCUMENT_XML], + ["word/comments.xml", COMMENTS_XML], + ["word/footnotes.xml", FOOTNOTES_XML], + ["word/endnotes.xml", ENDNOTES_XML] + ])); + const result = spawnSync(process.execPath, [join(ROOT, "scripts", "weaver.mjs"), "import", source, "--root", root], { cwd: ROOT, encoding: "utf8" }); + assert.equal(result.status, 0, `${result.stderr}\n${result.stdout}`); + assert.match(result.stdout, /Not imported: 3 comments, 1 footnote\./); + const record = JSON.parse(readFileSync(join(root, "imports", readdirSync(join(root, "imports"))[0]), "utf8")); + assert.deepEqual(record.not_imported, { comments: 3, footnotes: 1, endnotes: 0 }); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("escaped stars do not change word counts", () => { + assert.equal(countWords("f\\*\\*\\*"), 1); +}); + +test("import/export round trip is byte-safe for scene prose and leaves source untouched", () => { + const root = book(); + const second = book(); + try { + const source = fixture(root); + const before = readFileSync(source); + const mtime = statSync(source).mtimeMs; + const imported = importBook(root, readProject(root), source, { importedAt: "2026-09-25T00:00:00.000Z" }); + assert.equal(Buffer.compare(before, readFileSync(source)), 0); + assert.equal(statSync(source).mtimeMs, mtime); + assert.equal(imported.record.totals.scenes, 3); + const exported = join(root, "repaired.docx"); + exportBook(root, readProject(root), "docx", exported); + assert.equal(readFileSync(exported).subarray(0, 4).toString("hex"), "504b0304"); + importBook(second, readProject(second), exported, { importedAt: "2026-09-25T00:00:00.000Z" }); + assert.deepEqual(loadScenes(root, readProject(root)).map((scene) => scene.prose), loadScenes(second, readProject(second)).map((scene) => scene.prose)); + assert.throws(() => exportBook(root, readProject(root), "docx", exported), /overwrite/); + } finally { + rmSync(root, { recursive: true, force: true }); + rmSync(second, { recursive: true, force: true }); + } +}); + +test("import refuses existing scenes and the CLI doctor gate refuses a non-Git book", () => { + const existing = book(false); + const noGit = book(); + try { + const source = fixture(existing); + assert.throws(() => importBook(existing, readProject(existing), source), /already contains scene files/); + assert.equal(existsSync(join(existing, "imports")), false); + rmSync(join(noGit, ".git"), { recursive: true, force: true }); + const result = spawnSync(process.execPath, [join(ROOT, "scripts", "weaver.mjs"), "import", source, "--root", noGit], { + cwd: ROOT, + encoding: "utf8" + }); + assert.equal(result.status, 1); + assert.match(result.stdout, /Your book folder is not under Git/); + assert.equal(existsSync(join(noGit, "imports")), false); + } finally { + rmSync(existing, { recursive: true, force: true }); + rmSync(noGit, { recursive: true, force: true }); + } +}); + +test("Markdown and folder sources are deterministic in chapter order", () => { + const root = book(); + const source = mkdtempSync(join(tmpdir(), "weaver-source-")); + try { + writeFileSync(join(source, "chapter-10.txt"), "Chapter 10\n\nTen."); + writeFileSync(join(source, "chapter-2.md"), "# Wren\n\nTwo."); + const parsed = readSource(source); + assert.deepEqual(parsed.chapters.map((chapter) => chapter.title), ["Wren", ""]); + assert.deepEqual(parsed.chapters.map((chapter) => chapter.pov), ["Wren", "Wren"]); + importBook(root, readProject(root), source, { importedAt: "2026-09-25T00:00:00.000Z" }); + assert.deepEqual(loadScenes(root, readProject(root)).map((scene) => scene.chapter), [1, 2]); + } finally { + rmSync(root, { recursive: true, force: true }); + rmSync(source, { recursive: true, force: true }); + } +});