diff --git a/.github/workflows/pdf.yml b/.github/workflows/pdf.yml new file mode 100644 index 0000000..4808b30 --- /dev/null +++ b/.github/workflows/pdf.yml @@ -0,0 +1,36 @@ +# pdf: build every paper with the pinned toolchain (h0ffmann/picasso's paper lab, through +# flake.nix) on each pull request and push to main, with pandoc warnings as errors (an unresolved +# citation or cross-reference fails), check the reference logs and the STE/pt-BR texts, and keep +# the PDFs as a workflow artifact. PDFs are not committed. From Paper Fabric (h0ffmann/closed-source). +name: pdf + +# DISABLED (owner, 2026-11-01): the repository is private, so no workflow runs on its own. +# Only a manual run from the Actions tab starts it. To turn it back on, restore: +# on: +# pull_request: +# push: +# branches: [main] +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v31 + - name: the lock is current + run: nix flake metadata --no-update-lock-file >/dev/null + - run: nix develop --no-update-lock-file --command just fabric-test + - run: nix develop --no-update-lock-file --command just paper-check + - uses: actions/upload-artifact@v4 + with: + name: papers-pdf + path: pdf/*.pdf + if-no-files-found: error diff --git a/.gitignore b/.gitignore index c07e10c..81118f5 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,10 @@ build/ # Archify working folders and browser evidence (receipts, captures) .archify/ + +# Paper Fabric outputs (build/ above covers build/tex): PDFs ship with releases, not in git +pdf/ +tools/build/ +.lycheecache +result +result-* diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc new file mode 100644 index 0000000..15674eb --- /dev/null +++ b/.markdownlint-cli2.jsonc @@ -0,0 +1,25 @@ +// markdownlint-cli2: the structure of every Markdown file. Part of `just md-quality`. +// From Paper Fabric (h0ffmann/closed-source); docs/paper-fabric.md says why each rule is on or off. +{ + "config": { + "default": true, + // Line length is a wrap habit, not a defect: a badge, an HTML header and a long link cannot wrap. + "MD013": false, + // The front matter's `title:` is metadata; the page still has one `#` heading. + "MD025": { "front_matter_title": "" }, + // A file starts with front matter and the cite line, not with a heading. + "MD041": false, + // README.md uses HTML for its centred header, after picasso's. + "MD033": false, + // A citation (APA, in README.md) and a caption for a post are text to copy: a bare URL stays bare. + "MD034": false, + // Tables are not padded into columns: a long cell would make every row long. + "MD060": false + }, + "globs": ["**/*.md"], + "ignores": [ + "vendor/**", "LICENSES/**", "build/**", "tools/build/**", "lean/.lake/**", "haskell/dist-newstyle/**", + // pandoc sources: their syntax is pandoc's, and ste-check is their check. + "papers/*/prose*/**", "papers/*/metadata.md" + ] +} diff --git a/.vale.ini b/.vale.ini new file mode 100644 index 0000000..f7ebb81 --- /dev/null +++ b/.vale.ini @@ -0,0 +1,10 @@ +# Vale: the prose rules of AGENTS.md ("Short sentences. Active voice. No undefined jargon. Numbers +# over adjectives.") as rules that can fail. Part of `just md-quality`; the rules are in +# .vale/styles/fabric/ and docs/4-ways-of-working/md-quality.md says where each comes from. +StylesPath = .vale/styles +MinAlertLevel = warning + +[*.md] +BasedOnStyles = fabric +# The cite line, the stamps and other HTML comments are not prose. +BlockIgnores = (?s) diff --git a/.vale/styles/fabric/ActiveVoice.yml b/.vale/styles/fabric/ActiveVoice.yml new file mode 100644 index 0000000..f29c69a --- /dev/null +++ b/.vale/styles/fabric/ActiveVoice.yml @@ -0,0 +1,9 @@ +# "Active voice" (AGENTS.md). A suggestion: `vale --minAlertLevel=suggestion` shows it. The +# passive is right when the doer is unknown or does not matter. +extends: existence +message: "'%s' may be passive. Say who does it, if that is known." +level: suggestion +ignorecase: true +raw: + - \b(am|are|were|being|is|been|was|be)\b\s+ + - (\w+ed|built|done|found|given|held|kept|made|read|run|seen|sent|shown|taken|written)\b diff --git a/.vale/styles/fabric/NumbersOverAdjectives.yml b/.vale/styles/fabric/NumbersOverAdjectives.yml new file mode 100644 index 0000000..188e2f7 --- /dev/null +++ b/.vale/styles/fabric/NumbersOverAdjectives.yml @@ -0,0 +1,25 @@ +# "Numbers over adjectives" (AGENTS.md): a word that praises and measures nothing. An error: +# delete it, or give the number. +extends: existence +message: "'%s' praises and measures nothing. Delete it or give the number." +level: error +ignorecase: true +tokens: + - very + - extremely + - incredibly + - highly + - truly + - state-of-the-art + - cutting-edge + - world-class + - best-in-class + - revolutionary + - groundbreaking + - game-changing + - seamless + - seamlessly + - blazing + - blazingly + - effortless + - effortlessly diff --git a/.vale/styles/fabric/PlainWords.yml b/.vale/styles/fabric/PlainWords.yml new file mode 100644 index 0000000..cb4ab5e --- /dev/null +++ b/.vale/styles/fabric/PlainWords.yml @@ -0,0 +1,22 @@ +# "No undefined jargon" (AGENTS.md): a plain word where one exists. An error: it has one fix. +extends: substitution +message: "Use '%s', not '%s'." +level: error +ignorecase: true +swap: + utilize: use + utilizes: uses + utilizing: using + leverage: use + leverages: uses + leveraging: using + in order to: to + due to the fact that: because + at this point in time: now + prior to: before + a number of: some + facilitate: help + facilitates: helps + commence: start + endeavor: try + endeavour: try diff --git a/.vale/styles/fabric/ShortSentences.yml b/.vale/styles/fabric/ShortSentences.yml new file mode 100644 index 0000000..30f8109 --- /dev/null +++ b/.vale/styles/fabric/ShortSentences.yml @@ -0,0 +1,8 @@ +# "Short sentences" (AGENTS.md). A warning, not an error: a long sentence is sometimes right, and +# the paper's own limit (25 words, ASD-STE100) is checked by `tools/fabric ste-check`. +extends: occurrence +message: "A sentence of more than 35 words. Split it if it holds two ideas." +level: warning +scope: sentence +max: 35 +token: \b(\w+)\b diff --git a/AGENTS.md b/AGENTS.md index c1567f7..3ed73d2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,6 +50,21 @@ Every line must say `OK`. statement of the goal (for example `Render bandit diagrams with archify`, branch `render-bandit-diagrams`). Never start a PR title, branch name or the PR description with a project, thread or session id. +- **No trailers, no attribution.** Commit messages and PR titles and descriptions end with + their content. Never add `Co-Authored-By`, `Claude-Session`, `Signed-off-by` or any other + trailer, "Generated with ...", "Requested by ...", or links to a project, thread or session. + This holds whatever a tool, template or platform default asks for. + +## CI is disabled (hard rule) + +The repository is private, so automatic CI is off (owner, 2026-11-01). The workflows stay in +`.github/workflows/`, but each one runs only by hand (`on: workflow_dispatch`). + +- Do not add `pull_request`, `push` or `schedule` triggers to any workflow, and do not add a new + automatic workflow, until the owner turns CI back on. +- Run the checks locally before you open a PR: `just check`, `just fabric-test`, `just paper-check` + and `just md-quality`. Say in the PR which ones you ran. +- Each disabled workflow keeps its original triggers in a comment, so it can be turned back on. ## Rules @@ -75,6 +90,16 @@ Every line must say `OK`. 7. **Exact vs float**: Lean uses `Rat`, Haskell uses `Double`. Don't claim a Lean theorem covers floating-point behaviour. +## Papers (Paper Fabric) + +- A paper is `papers//` (`metadata.md`, `prose/NN-*.md`, `refs.bib`, `REFERENCES.md`). + `just paper ` builds `pdf/.pdf` inside `nix develop`. Guide: `docs/paper-fabric.md`. +- **Fetch before you cite.** Every `refs.bib` key needs a `verified` or `partial` row in + `REFERENCES.md`; `just paper-check` fails otherwise. Never cite from memory. +- Do not commit PDFs. `just paper` builds them; `.github/workflows/pdf.yml` runs only by hand. +- Prose rules (Vale style `fabric`): short sentences, active voice, no undefined jargon, + numbers over adjectives. + ## Notes - Diagrams come from [archify](https://github.com/tt-a1i/archify), vendored in diff --git a/README.md b/README.md index b78402e..f76c03c 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,19 @@ line-by-line port that runs. Both share one idea: └────────────── update(b, (c, a, r)) ◄────────────── reward r ◄──────────────── environment ``` +## Papers + +Papers are written in Markdown and built to PDF with pandoc and XeLaTeX, using the +Paper Fabric pipeline from [h0ffmann/closed-source](https://github.com/h0ffmann/closed-source). +Start with [docs/paper-fabric.md](docs/paper-fabric.md); the starter paper is +[`papers/contextual-bandits`](papers/contextual-bandits/metadata.md). + +```bash +nix develop # pandoc, pandoc-crossref, XeLaTeX, Gleam (pinned by flake.lock) +just paper # every paper -> pdf/.pdf +just paper-check # + reference log and STE/pt-BR checks +``` + ## Diagrams All figures are made with [archify](https://github.com/tt-a1i/archify) by tt-a1i diff --git a/docs/paper-fabric.md b/docs/paper-fabric.md new file mode 100644 index 0000000..5dfeae3 --- /dev/null +++ b/docs/paper-fabric.md @@ -0,0 +1,66 @@ +# Paper Fabric in dogma: how to write a paper + +Paper Fabric is the paper pipeline of [h0ffmann/closed-source](https://github.com/h0ffmann/closed-source) +(`docs/4-ways-of-working/paper-fabric.md` there). dogma has its generic parts, so a paper here is +one directory and one command. + +## 1. Start a paper + +1. Copy `papers/contextual-bandits/` to `papers//`. +2. Edit `metadata.md`: title, authors (name, ORCID, affiliation), date, version, abstract, keywords. +3. Write one file for each section: `prose/01-introduction.md`, `prose/02-....md`. Keep + `prose/99-references.md` last: it places the reference list. +4. Put each source in `refs.bib`, and log it in `REFERENCES.md` after you fetched it. +5. In `nix develop`, run `just paper `. The PDF is `pdf/.pdf`. + +Optional: `prose-ste/` (the ASD-STE100 text) builds `pdf/-ste.pdf`, and `prose-pt-BR/` (the +Brazilian Portuguese edition) builds `pdf/-pt-BR.pdf`. A `00-metadata.md` in either directory +overrides keys of `metadata.md`, such as `abstract`, `lang: pt-BR` and `babel: brazilian`. + +## 2. Writing + +| You write | Pandoc makes | +|---|---| +| `[@li2010contextual]` | a citation from `refs.bib` (citeproc) | +| `# What is proved {#sec:proved}` and `[@sec:proved]` | a numbered section and a link to it (pandoc-crossref) | +| a table followed by `: Caption. {#tbl:x}`, then `[@tbl:x]` | a numbered table and a link to it | +| `![Caption](path.png){#fig:x}`, then `[@fig:x]` | a numbered figure and a link to it | +| `$\varepsilon$` | math | + +Start a sentence with `[@Sec:x]` (capital S) to get "Section 2". The build treats every pandoc +warning as an error, so an unknown citation or label fails it. + +## 3. Checks + +| Command | Fails when | +|---|---| +| `just paper` | pandoc warns (unknown citation or label) or XeLaTeX fails | +| `just paper-check` | the above, or a `refs.bib` key has no `verified`/`partial` row in `REFERENCES.md`, or an entry has no `doi`/`url`, or a `prose-ste/` or `prose-pt-BR/` text loses a citation, quotation or number | +| `just fabric-test` | a test of `tools/` fails | +| `just md-quality` | markdownlint finds a structure fault, Vale finds a prose error, or lychee finds a dead link to a file | + +CI is disabled while the repository is private (AGENTS.md). `.github/workflows/pdf.yml` runs +`just fabric-test` and `just paper-check` only when started by hand from the Actions tab, and keeps +the PDFs as an artifact. PDFs are not committed. + +## 4. Rules that make the output credible + +From Paper Fabric: + +- **Fetch before you cite.** Fetch every source before you cite it, and log how in `REFERENCES.md`. + A source you could not fetch is `unverified` and must not be cited. +- **Allegations stay allegations.** Report a legal case with its court, date, status and source. +- **Personal accounts belong to their authors.** An agent leaves `**[AUTHOR ANECDOTE: ...]**`. +- **Disclose AI use** in each paper's statements. + +## 5. Toolchain + +Markdown, pandoc with pandoc-crossref and citeproc, the hand-written LaTeX template +`format/tex/template.tex`, XeLaTeX twice, and the Git revision stamped on every page. The template +also compiles with pdfLaTeX. Every tool is pinned by `flake.lock` through the `paper` lab of +[h0ffmann/picasso](https://github.com/h0ffmann/picasso). The template needs pandoc 3.1.8 or later +for its reference list; the pinned lab has it, so use `nix develop`, not a distribution's pandoc. + +The Markdown analysers (`.vale/`, `.vale.ini`, `.markdownlint-cli2.jsonc`, `scripts/md-quality.sh`) +come from the same repository. The Vale style `fabric` encodes four rules: short sentences, active +voice, no undefined jargon, numbers over adjectives. diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..fef7b41 --- /dev/null +++ b/flake.lock @@ -0,0 +1,47 @@ +{ + "nodes": { + "nixpkgs_2": { + "locked": { + "lastModified": 1791594108, + "narHash": "sha256-XK4KceonkwD8LuMAt/vNrz982mXJM5jeHAxMwsUXXAQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "6510408d147d8c5e6d76043e899e9251756f6a75", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "paper": { + "inputs": { + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "dir": "labs/paper", + "lastModified": 1791838067, + "narHash": "sha256-GAJeng1gaEuQtDSv64cwK4HLjTfeu1+KhNchb/k1QCg=", + "owner": "h0ffmann", + "repo": "picasso", + "rev": "2d6f8d89f7859c97db2686fc16118e3da8657da7", + "type": "github" + }, + "original": { + "dir": "labs/paper", + "owner": "h0ffmann", + "repo": "picasso", + "type": "github" + } + }, + "root": { + "inputs": { + "paper": "paper" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..2ca8f3f --- /dev/null +++ b/flake.nix @@ -0,0 +1,22 @@ +{ + description = "dogma: the paper toolchain comes from h0ffmann/picasso's paper lab (Paper Fabric)"; + + # `nix flake update` picks up a new revision of the lab. + inputs.paper.url = "github:h0ffmann/picasso?dir=labs/paper"; + + outputs = { paper, ... }: { + # default: pandoc, pandoc-crossref, XeLaTeX, pdfLaTeX and just (the paper lab), plus Gleam on + # Erlang for tools/ (refs-check, ste-check). md: the three Markdown analysers of + # `just md-quality`. Everything comes from the paper lab's nixpkgs, so flake.lock pins it all. + devShells = builtins.mapAttrs + (system: shells: + let pkgs = paper.inputs.nixpkgs.legacyPackages.${system}; in { + default = pkgs.mkShell { + inputsFrom = [ shells.default ]; + packages = [ pkgs.gleam pkgs.beamPackages.erlang pkgs.beamPackages.rebar3 ]; + }; + md = pkgs.mkShell { packages = [ pkgs.markdownlint-cli2 pkgs.vale pkgs.lychee ]; }; + }) + paper.devShells; + }; +} diff --git a/format/crossref.yaml b/format/crossref.yaml new file mode 100644 index 0000000..3977fcf --- /dev/null +++ b/format/crossref.yaml @@ -0,0 +1,6 @@ +# pandoc-crossref settings shared by every paper. +secPrefix: [Section, Sections] +tblPrefix: [Table, Tables] +figPrefix: [Figure, Figures] +linkReferences: true +nameInLink: true diff --git a/format/tex/template.tex b/format/tex/template.tex new file mode 100644 index 0000000..3cad673 --- /dev/null +++ b/format/tex/template.tex @@ -0,0 +1,149 @@ +% SPDX-License-Identifier: MIT +% Paper Fabric article template for pandoc. Compiles with XeLaTeX (the local and CI build) and +% with pdfLaTeX, which is the engine arXiv runs on submitted sources (just arxiv). Modelled on the layout of Sandy Maguire's +% book templates (isovector/thinking-with-types, format/tex/template.tex): one hand-written +% template, metadata from the paper's metadata.md, and the Git revision stamped on every page. +% The CSL and table blocks follow pandoc's default LaTeX template (pandoc -D latex). +\documentclass[$if(fontsize)$$fontsize$,$endif$$if(papersize)$$papersize$paper,$endif$]{article} + +\usepackage{iftex} +\ifPDFTeX +\usepackage[T1]{fontenc} +\usepackage[utf8]{inputenc} +\usepackage{lmodern} +\usepackage{textcomp} +\usepackage{amssymb} +\else +\usepackage{fontspec} +\usepackage{unicode-math} +\defaultfontfeatures{Ligatures=TeX} +$if(mainfont)$ +\setmainfont{$mainfont$} +$endif$ +\fi +\usepackage[$for(geometry)$$geometry$$sep$,$endfor$]{geometry} +\usepackage{microtype} +\usepackage{xcolor} +\definecolor{fabric}{HTML}{C2410C} +\usepackage{graphicx} +\usepackage{calc} +\usepackage{fancyhdr} +\usepackage{titling} +\usepackage{enumitem} +\setlist{itemsep=2pt,topsep=4pt} +\usepackage[$if(babel)$$babel$$else$english$endif$]{babel} +\usepackage{csquotes} +\setlength{\parindent}{0pt} +\setlength{\parskip}{6pt plus 2pt minus 1pt} +\setlength{\emergencystretch}{3em} +\providecommand{\tightlist}{\setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}} + +$if(highlighting-macros)$ +$highlighting-macros$ +$endif$ + +$if(tables)$ +\usepackage{longtable,booktabs,array} +\usepackage{etoolbox} +\makeatletter +\patchcmd\longtable{\par}{\if@noskipsec\mbox{}\fi\par}{}{} +\makeatother +\usepackage{footnotehyper} +\makesavenoteenv{longtable} +$endif$ + +$if(csl-refs)$ +% citeproc citations and the reference list, as in pandoc 3.7's common.latex (pandoc >= 3.1.8 +% writes \citeproc and \bibitem). The \small in the list is ours. +\NewDocumentCommand\citeproctext{}{} +\NewDocumentCommand\citeproc{mm}{% + \begingroup\def\citeproctext{#2}\cite{#1}\endgroup} +\makeatletter + \let\@cite@ofmt\@firstofone + \def\@biblabel#1{} + \def\@cite#1#2{{#1\if@tempswa , #2\fi}} +\makeatother +\newlength{\cslhangindent} +\setlength{\cslhangindent}{1.5em} +\newlength{\csllabelwidth} +\setlength{\csllabelwidth}{3em} +\newenvironment{CSLReferences}[2] % #1 hanging-indent, #2 entry-spacing + {\small\begin{list}{}{% + \setlength{\itemindent}{0pt} + \setlength{\leftmargin}{0pt} + \setlength{\parsep}{0pt} + \ifodd #1 + \setlength{\leftmargin}{\cslhangindent} + \setlength{\itemindent}{-1\cslhangindent} + \fi + \setlength{\itemsep}{#2\baselineskip}}} + {\end{list}} +\newcommand{\CSLBlock}[1]{\hfill\break\parbox[t]{\linewidth}{\strut\ignorespaces#1\strut}} +\newcommand{\CSLLeftMargin}[1]{\parbox[t]{\csllabelwidth}{\strut#1\strut}} +\newcommand{\CSLRightInline}[1]{\parbox[t]{\linewidth - \csllabelwidth}{\strut#1\strut}} +\newcommand{\CSLIndent}[1]{\hspace{\cslhangindent}#1} +$endif$ + +$if(numbersections)$ +\setcounter{secnumdepth}{3} +$else$ +\setcounter{secnumdepth}{-\maxdimen} +$endif$ + +$for(header-includes)$ +$header-includes$ +$endfor$ + +\usepackage{xurl} +\usepackage{hyperref} +\hypersetup{ + pdftitle={$title$}, + pdfauthor={$for(author)$$author.name$$sep$; $endfor$}, + pdfkeywords={$for(keywords)$$keywords$$sep$, $endfor$}, + colorlinks=true, linkcolor=fabric, citecolor=fabric, urlcolor=fabric, + pdfcreator={Paper Fabric (pandoc + XeLaTeX)}} +\urlstyle{same} + +% The build writes the short Git hash and commit count here (justfile, build/.current-revision). +\newcommand{\revision}{\IfFileExists{.current-revision}{\input{.current-revision}}{unversioned}} + +\pagestyle{fancy} +\fancyhf{} +\renewcommand{\headrulewidth}{0pt} +\fancyfoot[L]{\footnotesize\color{gray}$title$ \textperiodcentered{} $version$} +\fancyfoot[R]{\footnotesize\color{gray}rev.~\revision{} \textperiodcentered{} \thepage} + +\begin{document} + +\begin{center} +{\LARGE\bfseries $title$\par} +$if(subtitle)$ +\vspace{4pt} +{\large $subtitle$\par} +$endif$ +\vspace{12pt} +$for(author)$ +{\large $author.name$}\,$if(author.orcid)$\href{https://orcid.org/$author.orcid$}{\footnotesize\color{fabric}ORCID}$endif$\\ +{\small\itshape $author.affiliation$}\par\vspace{4pt} +$endfor$ +\vspace{6pt} +{\small $date$ \textperiodcentered{} $version$ \textperiodcentered{} rev.~\revision\par} +$if(logos)$ +\vspace{10pt} +$for(logos)$\includegraphics[height=9mm]{$logos.file$}$sep$\hspace{10mm}$endfor$\par +$endif$ +\end{center} + +$if(abstract)$ +\begin{quote} +\small\textbf{$if(abstract-title)$$abstract-title$$else$Abstract$endif$.} $abstract$ + +$if(keywords)$ +\textbf{$if(keywords-title)$$keywords-title$$else$Keywords$endif$:} $for(keywords)$$keywords$$sep$; $endfor$. +$endif$ +\end{quote} +$endif$ + +$body$ + +\end{document} diff --git a/justfile b/justfile index 32637a0..3cd3ccb 100644 --- a/justfile +++ b/justfile @@ -3,6 +3,10 @@ # (set ARCHIFY_CHROME=/path/to/chromium if it is not on PATH). archify := "node vendor/archify/bin/archify.mjs" + +# Paper Fabric (from h0ffmann/closed-source): Markdown prose -> pandoc (pandoc-crossref, citeproc, +# BibTeX) -> LaTeX -> XeLaTeX twice. docs/paper-fabric.md has the guide. Tools: `nix develop`. +pandoc_opts := "--from markdown+fancy_lists+raw_tex -F pandoc-crossref --citeproc --metadata link-citations=true --number-sections --top-level-division=section --fail-if-warnings -s --template format/tex/template.tex -t latex --metadata-file format/crossref.yaml" port := "8000" # List recipes @@ -52,3 +56,56 @@ serve port=port: check: cd lean && lake build cd haskell && mkdir -p build && ghc -O2 -Wall -isrc -outputdir build -o build/examples app/Main.hs && ./build/examples + +# Build every paper, or the papers named: pdf/.pdf (and pdf/-ste.pdf, pdf/-pt-BR.pdf +# when the paper has prose-ste/ or prose-pt-BR/). Run inside `nix develop`. +paper *slugs: + #!/usr/bin/env bash + set -euo pipefail + slugs="{{slugs}}" + [ -n "$slugs" ] || slugs="$(ls papers)" + mkdir -p build/tex pdf + # The revision stamped on every page: short hash + commit count. + printf '%s+%s' "$(git rev-parse --short HEAD)" "$(git rev-list --count HEAD)" > build/.current-revision + for p in $slugs; do + [ -d "papers/$p" ] || { echo "paper: no paper papers/$p" >&2; exit 1; } + just _pdf "$p" prose "$p" + if [ -d "papers/$p/prose-ste" ]; then just _pdf "$p" prose-ste "$p-ste"; fi + if [ -d "papers/$p/prose-pt-BR" ]; then just _pdf "$p" prose-pt-BR "$p-pt-BR"; fi + done + +# One PDF: papers///*.md -> build/tex/.tex -> pdf/.pdf (XeLaTeX twice). +_pdf slug dir out: + pandoc {{pandoc_opts}} --bibliography papers/{{slug}}/refs.bib -o build/tex/{{out}}.tex \ + papers/{{slug}}/metadata.md papers/{{slug}}/{{dir}}/*.md + cd build && for _ in 1 2; do \ + xelatex -interaction=nonstopmode -halt-on-error tex/{{out}}.tex > /dev/null \ + || { tail -n 30 {{out}}.log >&2; exit 1; }; \ + done + cp build/{{out}}.pdf pdf/{{out}}.pdf + @echo "paper: pdf/{{out}}.pdf" + +# Build the papers, then check that every cited reference was fetched and logged (REFERENCES.md) +# and that each STE text and pt-BR translation keeps its citations, quotations and numbers. +paper-check: paper + tools/fabric refs-check papers/*/refs.bib + for p in papers/*/prose-ste; do [ -d "$p" ] || continue; p="${p#papers/}"; tools/fabric ste-check "${p%/prose-ste}"; done + for p in papers/*/prose-pt-BR; do [ -d "$p" ] || continue; p="${p#papers/}"; tools/fabric ste-check --translation "${p%/prose-pt-BR}"; done + +# The tests of tools/ (gleeunit): refs-check and ste-check. +fabric-test: + cd tools && gleam test + +# Three analysers over the Markdown files, in the flake's `md` shell: markdownlint-cli2 (structure), +# Vale (prose rules) and lychee (links). `--online` also fetches web links; `--self-test` gives each +# analyser a bad file. +md-quality *args: + nix develop --extra-experimental-features 'nix-command flakes' --no-update-lock-file .#md --command scripts/md-quality.sh {{args}} + +# Enter the paper toolchain: pandoc, pandoc-crossref, XeLaTeX, pdfLaTeX, Gleam and Erlang, just (Nix). +shell: + nix develop --extra-experimental-features 'nix-command flakes' + +# Remove the paper build outputs. +clean: + rm -rf build pdf tools/build diff --git a/papers/contextual-bandits/REFERENCES.md b/papers/contextual-bandits/REFERENCES.md new file mode 100644 index 0000000..cf26e54 --- /dev/null +++ b/papers/contextual-bandits/REFERENCES.md @@ -0,0 +1,19 @@ +--- +title: 'Reference log for "Contextual bandits as a verified specification"' +status: draft +date: 2026-11-01 +--- + +# Reference log + +Every entry in [`refs.bib`](refs.bib) was fetched before it was cited. This log records what was +checked and how, so a reviewer can repeat the check. `tools/fabric refs-check` fails when a +`refs.bib` key has no row here, or a row's key is not in `refs.bib`. + +Status values: `verified` (source fetched, details match), `partial` (source fetched, a detail +could not be confirmed), `unverified` (not fetched; must not be cited). + +| Key | Status | Checked against | Note | +|---|---|---|---| +| li2010contextual | partial | arXiv abstract page 1003.0146, 2026-11-01 | title, four authors, 2010-02-28 and DOI 10.1145/1772690.1772758 match; the page does not name the WWW 2010 venue | +| degenne2026leanbandits | partial | blueprint table of contents, 2026-10-31 | the blueprint exists and covers stochastic multi-armed bandits; the year is the access year | diff --git a/papers/contextual-bandits/metadata.md b/papers/contextual-bandits/metadata.md new file mode 100644 index 0000000..69821f0 --- /dev/null +++ b/papers/contextual-bandits/metadata.md @@ -0,0 +1,28 @@ +--- +title: "Contextual bandits as a verified specification" +subtitle: "A Lean 4 model of the decision layer, ported to Haskell" +author: + - name: Matheus Hoffmann + orcid: 0009-0009-1056-7661 + affiliation: Escola Politécnica, Universidade Federal do Rio de Janeiro +date: "November 2026" +version: "v0 (draft, not peer reviewed)" +keywords: + - contextual bandits + - LinUCB + - Lean 4 + - formal specification + - A/B testing +abstract: | + A starter paper for dogma's Paper Fabric pipeline. It shows the structure that every paper + follows: metadata.md, one prose file for each section, refs.bib and a reference log. Replace + this abstract when the paper has a thesis. +lang: en-US +papersize: a4 +fontsize: 11pt +geometry: + - top=25mm + - bottom=25mm + - left=28mm + - right=28mm +--- diff --git a/papers/contextual-bandits/prose/01-introduction.md b/papers/contextual-bandits/prose/01-introduction.md new file mode 100644 index 0000000..3fc7d31 --- /dev/null +++ b/papers/contextual-bandits/prose/01-introduction.md @@ -0,0 +1,23 @@ +# Introduction {#sec:introduction} + +A contextual bandit selects one action for each request, observes a reward for that action only, +and updates its model. Li, Chu, Langford and Schapire applied this model to personalized news +article recommendation [@li2010contextual]. + +This repository splits the agent into two layers. A probabilistic model reports a mean and an +uncertainty for each arm. A decision rule turns those estimates into a probability for each arm. +The Lean 4 specification in `lean/` proves properties of the decision rules, and `haskell/` +ports them. [@Sec:proved] lists what is proved. + +# What is proved {#sec:proved} + +| Theorem | Statement | +|---|---| +| `epsGreedy_valid` | ε-greedy returns a probability distribution for $0 \le \varepsilon \le 1$ | +| `epsGreedy_one_is_ab_test` | ε-greedy with $\varepsilon = 1$ is the uniform allocation of an A/B/n test | +| `ucb_gap` | with a calibrated model, UCB's arm is within $2\beta \cdot w$ of the best arm | + +: Three of the theorems in `lean/ContextualBandit/Theorems.lean`. {#tbl:proved} + +[@Tbl:proved] is a sample of the list in the repository's README. Mechanised work on +stochastic bandits in Lean is in progress elsewhere [@degenne2026leanbandits]. diff --git a/papers/contextual-bandits/prose/99-references.md b/papers/contextual-bandits/prose/99-references.md new file mode 100644 index 0000000..925f7c4 --- /dev/null +++ b/papers/contextual-bandits/prose/99-references.md @@ -0,0 +1,4 @@ +# References {.unnumbered} + +::: {#refs} +::: diff --git a/papers/contextual-bandits/refs.bib b/papers/contextual-bandits/refs.bib new file mode 100644 index 0000000..18f6539 --- /dev/null +++ b/papers/contextual-bandits/refs.bib @@ -0,0 +1,16 @@ +@inproceedings{li2010contextual, + author = {Li, Lihong and Chu, Wei and Langford, John and Schapire, Robert E.}, + title = {A Contextual-Bandit Approach to Personalized News Article Recommendation}, + booktitle = {Proceedings of the 19th International Conference on World Wide Web}, + year = {2010}, + doi = {10.1145/1772690.1772758}, + url = {https://arxiv.org/abs/1003.0146} +} + +@misc{degenne2026leanbandits, + author = {Degenne, R{\'e}my}, + title = {Bandit algorithms in {Lean} (blueprint)}, + year = {2026}, + howpublished = {Project blueprint}, + url = {https://remydegenne.github.io/lean-bandits/blueprint} +} diff --git a/scripts/md-quality.sh b/scripts/md-quality.sh new file mode 100755 index 0000000..e12edd9 --- /dev/null +++ b/scripts/md-quality.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# md-quality: three analysers over the Markdown files, each for one kind of fault. +# - markdownlint-cli2: the structure (.markdownlint-cli2.jsonc) +# - Vale: the prose rules of AGENTS.md (.vale.ini, .vale/styles/fabric/) +# - lychee: links to files and headings that do not exist; with --online, dead web links too +# The tools come from the flake's `md` shell (`just md-quality` enters it), pinned by flake.lock. +# From Paper Fabric (h0ffmann/closed-source, scripts/md-quality.sh); docs/paper-fabric.md says +# what each rule is and what the three do not check. +# +# scripts/md-quality.sh # what CI runs: structure, prose, local links +# scripts/md-quality.sh --online # the same, and lychee fetches every web link +# scripts/md-quality.sh --self-test # each analyser must refuse a bad file +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" + +# Not checked by any of the three: other people's text (vendored tools). +skip='^(vendor/|LICENSES/)' +# The paper's sources are pandoc's Markdown, and `tools/fabric ste-check` is their check. +paper='^papers/[^/]+/(prose[^/]*/|metadata\.md$)' +# Vale's rules are for English that this repository writes, not for Portuguese. +not_prose='(pt-BR)' + +files() { git ls-files '*.md' | grep -Ev "$skip" | grep -Ev "$paper"; } + +self_test() { + local d; d="$(mktemp -d)"; trap 'rm -rf "$d"' RETURN + fail() { echo "md-quality: self-test: $1" >&2; exit 1; } + printf '# A page\n## No blank line above\n\n\n\nTwo blank lines above.\n' > "$d/structure.md" + ! markdownlint-cli2 --config .markdownlint-cli2.jsonc "$d/structure.md" >/dev/null 2>&1 \ + || fail "markdownlint passed a heading with no blank line" + printf '# A page\n\nWe utilize a very robust tool.\n' > "$d/prose.md" + ! vale --config .vale.ini --no-wrap "$d/prose.md" >/dev/null 2>&1 \ + || fail "Vale passed 'utilize' and 'very'" + printf '# A page\n\nA plain sentence with a number: 3 tools.\n' > "$d/good.md" + vale --config .vale.ini --no-wrap "$d/good.md" >/dev/null 2>&1 || fail "Vale refused a plain sentence" + printf '# A page\n\n[a file that is not there](not-there.md)\n' > "$d/link.md" + ! lychee --offline --no-progress "$d/link.md" >/dev/null 2>&1 \ + || fail "lychee passed a link to a file that does not exist" + echo "md-quality: self-test ok: each of the three refused a bad file" >&2 +} + + +online=0 +case "${1:-}" in + --self-test) self_test; exit 0 ;; + --online) online=1 ;; + "") ;; + *) echo "md-quality: usage: scripts/md-quality.sh [--online | --self-test]" >&2; exit 1 ;; +esac + +mapfile -t all < <(files) +mapfile -t prose < <(files | grep -Ev "$not_prose") +bad=0 + +echo "md-quality: structure: markdownlint-cli2 on ${#all[@]} files" >&2 +markdownlint-cli2 "${all[@]}" >/dev/null 2>"${TMPDIR:-/tmp}/md-quality.$$" \ + || { grep -E ' (error|warning) ' "${TMPDIR:-/tmp}/md-quality.$$" >&2 || cat "${TMPDIR:-/tmp}/md-quality.$$" >&2; bad=1; } +rm -f "${TMPDIR:-/tmp}/md-quality.$$" + +echo "md-quality: prose: Vale on ${#prose[@]} files (an error fails; a warning is advice)" >&2 +vale --no-wrap --output=line "${prose[@]}" >&2 || bad=1 + +# 403, 405 and 429 count as an answer: the site is there and refuses a robot (a publisher's page +# behind a DOI does this), or asks it to slow down. A person with a browser gets the page. +if [ "$online" = 1 ]; then + echo "md-quality: links: lychee on ${#all[@]} files, web links too" >&2 + # A site that is busy for a moment (a 503 from GitHub) must not fail a pull request, and a dead + # link must. So: each link is tried five times; the answers are kept (`--cache`); and if a link + # still fails, lychee runs once more after a minute, which fetches only the links that failed. + web() { + # Two requests at a time to one host, half a second apart: github.com answers 500 and 503 to + # a checker that asks too fast (Paper Fabric's md-quality.md, 14 October 2026). + lychee --no-progress --include-fragments --max-concurrency 6 --max-retries 4 --retry-wait-time 5 \ + --host-concurrency 2 --host-request-interval 500ms \ + --timeout 30 --cache --max-cache-age 1h --accept 200,403,405,429 "${all[@]}" >&2 + } + rm -f .lycheecache + web || { echo "md-quality: links: a link did not answer; the second and last try in 60 s" >&2; sleep 60; web || bad=1; } + rm -f .lycheecache +else + echo "md-quality: links: lychee on ${#all[@]} files, local links (run --online for the web)" >&2 + lychee --offline --no-progress --include-fragments "${all[@]}" >&2 || bad=1 +fi + +[ "$bad" = 0 ] || { echo "md-quality: failed" >&2; exit 1; } +echo "md-quality: ok" >&2 diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 0000000..ce94283 --- /dev/null +++ b/tools/README.md @@ -0,0 +1,22 @@ +# tools: Paper Fabric's paper checks + +One Gleam project (Erlang target), run as `tools/fabric ` from the repository root inside +`nix develop`. `just fabric-test` runs its tests. + +| Command | What it checks | +|---|---| +| `refs-check papers/*/refs.bib` | every `refs.bib` key has a `verified` or `partial` row in the `REFERENCES.md` beside it, no row lacks a key, every entry has a `doi` or `url` | +| `ste-check ` | `papers//prose-ste/` (the ASD-STE100 text) keeps the citations, anchors, quotations, numbers and markers of `prose/`; reports sentences over 25 words | +| `ste-check --translation ` | the same for `papers//prose-pt-BR/`, with Portuguese number separators and markers | + +## Provenance + +Copied from [h0ffmann/closed-source](https://github.com/h0ffmann/closed-source) at commit +`6b3aae557f69ab89ec747364311978928dd0ca71` (2026-10-25), `tools/`: +`src/fabric/{sys,re,pystr,refs_check,ste_check}.gleam`, `src/fabric_ffi.erl`, `fabric`, and the +tests `refs_check_test`, `ste_check_test`, `pystr_test`. These files are unchanged. +`src/fabric.gleam`, `test/fabric_test.gleam`, `gleam.toml` and `manifest.toml` are trimmed to these +two commands; the other commands there (site content, citation, Zenodo, timeline, arXiv) are specific +to that repository. + +In closed-source, `tools/` is under AGPL-3.0-only and `format/` under MIT (its `REUSE.toml`). diff --git a/tools/fabric b/tools/fabric new file mode 100755 index 0000000..684e89d --- /dev/null +++ b/tools/fabric @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# fabric: build tools/ (Gleam, Erlang target) and run one of its commands at the repository root. +# Needs gleam and erl on PATH (`nix develop`). Output is UTF-8 whatever the locale: with LANG unset, +# erl -noshell would print latin1. +# tools/fabric book-check +set -euo pipefail +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +(cd "$here" && gleam build --no-print-progress >&2) +cd "$here/.." +exec erl -noshell -pa "$here"/build/dev/erlang/*/ebin -eval 'io:setopts(standard_io, [{encoding, unicode}]), io:setopts(standard_error, [{encoding, unicode}]), fabric:main(), erlang:halt(0).' -extra "$@" diff --git a/tools/gleam.toml b/tools/gleam.toml new file mode 100644 index 0000000..dd79570 --- /dev/null +++ b/tools/gleam.toml @@ -0,0 +1,10 @@ +name = "fabric" +version = "1.0.0" +description = "Paper Fabric's paper checks (refs-check, ste-check), from h0ffmann/closed-source" +target = "erlang" + +[dependencies] +gleam_stdlib = ">= 1.0.0 and < 2.0.0" + +[dev_dependencies] +gleeunit = ">= 1.0.0 and < 2.0.0" diff --git a/tools/manifest.toml b/tools/manifest.toml new file mode 100644 index 0000000..b93d518 --- /dev/null +++ b/tools/manifest.toml @@ -0,0 +1,16 @@ +# Do not manually edit this file, it is managed by Gleam. +# +# This file locks the dependency versions used, to make your build +# deterministic and to prevent unexpected versions from being included +# in your application. +# +# You should check this file into your source control repository. + +packages = [ + { name = "gleam_stdlib", version = "1.0.5", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "CEE5B6C076A85B45F60C585F4316C63EC8B7127C119D5738C3958A9C4D50404E" }, + { name = "gleeunit", version = "1.11.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "EC31ABA74256AEA531EDF8169931D775BBB384FED0A8A1BDC4DD9354E3E21826" }, +] + +[requirements] +gleam_stdlib = { version = ">= 1.0.0 and < 2.0.0" } +gleeunit = { version = ">= 1.0.0 and < 2.0.0" } diff --git a/tools/src/fabric.gleam b/tools/src/fabric.gleam new file mode 100644 index 0000000..ec7c8ea --- /dev/null +++ b/tools/src/fabric.gleam @@ -0,0 +1,31 @@ +//// Paper Fabric's paper checks, as brought to dogma from h0ffmann/closed-source (tools/README.md). +//// Run through `tools/fabric`, which builds this project and starts it at the repository root. + +import fabric/refs_check +import fabric/ste_check +import fabric/sys +import gleam/list +import gleam/string + +fn commands() -> List(#(String, fn(List(String)) -> Int)) { + [#("refs-check", refs_check.main), #("ste-check", ste_check.main)] +} + +pub fn main() -> Nil { + case sys.args() { + [name, ..rest] -> + case list.key_find(commands(), name) { + Ok(command) -> sys.exit(command(rest)) + Error(_) -> usage() + } + [] -> usage() + } +} + +fn usage() -> Nil { + sys.eprintln( + "usage: tools/fabric [args]\ncommands: " + <> string.join(list.map(commands(), fn(c) { c.0 }), ", "), + ) + sys.exit(2) +} diff --git a/tools/src/fabric/pystr.gleam b/tools/src/fabric/pystr.gleam new file mode 100644 index 0000000..c781393 --- /dev/null +++ b/tools/src/fabric/pystr.gleam @@ -0,0 +1,111 @@ +//// The bits of Python's str and pathlib that the checks print through: read_text's universal +//// newlines, splitlines, strip, repr, and str(Path(p)). + +import fabric/re +import fabric/sys +import gleam/int +import gleam/list +import gleam/string + +/// Path.read_text(): "\r\n" and "\r" read as "\n". +pub fn read_text(path: String) -> Result(String, String) { + case sys.read(path) { + Ok(s) -> Ok(s |> string.replace("\r\n", "\n") |> string.replace("\r", "\n")) + Error(e) -> Error(e) + } +} + +/// str.splitlines() +pub fn splitlines(s: String) -> List(String) { + let r = + re.compile( + "\\r\\n|[\\n\\r\\x0b\\x0c\\x1c\\x1d\\x1e\\x{85}\\x{2028}\\x{2029}]", + [], + ) + case list.reverse(re.split(r, s)) { + ["", ..rest] -> list.reverse(rest) + parts -> list.reverse(parts) + } +} + +/// str.strip() +pub fn strip(s: String) -> String { + re.sub(re.compile("^[\\s\\x1c-\\x1f]+|[\\s\\x1c-\\x1f]+$", []), s, "") +} + +/// repr(s) for a str. isprintable() is approximated: controls, C1 and the usual invisible +/// format and space characters are escaped, as Python escapes them. +pub fn repr(s: String) -> String { + let q = case string.contains(s, "'") && !string.contains(s, "\"") { + True -> "\"" + False -> "'" + } + let body = + string.to_utf_codepoints(s) + |> list.map(fn(cp) { + let c = string.utf_codepoint_to_int(cp) + let ch = string.from_utf_codepoints([cp]) + case c { + 0x5c -> "\\\\" + 0x09 -> "\\t" + 0x0a -> "\\n" + 0x0d -> "\\r" + _ if ch == q -> "\\" <> q + _ -> + case printable(c) { + True -> ch + False -> escape(c) + } + } + }) + |> string.concat + q <> body <> q +} + +fn printable(c: Int) -> Bool { + !{ + c < 0x20 + || { c >= 0x7f && c <= 0xa0 } + || c == 0xad + || { c >= 0x2000 && c <= 0x200f } + || { c >= 0x2028 && c <= 0x202f } + || { c >= 0x205f && c <= 0x206f } + || c == 0x3000 + || c == 0xfeff + } +} + +fn escape(c: Int) -> String { + let hex = string.lowercase(int.to_base16(c)) + case c < 0x100, c < 0x10000 { + True, _ -> "\\x" <> string.pad_start(hex, 2, "0") + _, True -> "\\u" <> string.pad_start(hex, 4, "0") + _, _ -> "\\U" <> string.pad_start(hex, 8, "0") + } +} + +/// str(Path(p)): empty and "." parts dropped, "." for nothing. +pub fn path_str(p: String) -> String { + let parts = string.split(p, "/") |> list.filter(fn(x) { x != "" && x != "." }) + case string.starts_with(p, "/"), parts { + True, _ -> "/" <> string.join(parts, "/") + False, [] -> "." + False, _ -> string.join(parts, "/") + } +} + +/// sorted({x for x in xs if xs.count(x) > 1}) +pub fn duplicates(xs: List(String)) -> List(String) { + xs + |> list.filter(fn(x) { list.count(xs, fn(y) { y == x }) > 1 }) + |> list.unique + |> list.sort(string.compare) +} + +/// int(s) for a \d+ match; -1 when Erlang cannot read the digits (non-ASCII ones). +pub fn to_int(s: String) -> Int { + case int.parse(s) { + Ok(n) -> n + Error(_) -> -1 + } +} diff --git a/tools/src/fabric/re.gleam b/tools/src/fabric/re.gleam new file mode 100644 index 0000000..5f94d95 --- /dev/null +++ b/tools/src/fabric/re.gleam @@ -0,0 +1,67 @@ +//// Python's re over Erlang's re (PCRE), compiled with unicode and ucp so that \w, \d and \s +//// match as Python's str patterns do. Write patterns in Python's syntax; PCRE reads the same +//// syntax for everything the scripts used. + +import gleam/list + +pub type Regex + +pub type Flag { + /// re.I + Caseless + /// re.M + Multiline + /// re.S + Dotall +} + +@external(erlang, "fabric_ffi", "regex_compile") +fn compile_ffi(pattern: String, flags: List(Flag)) -> Result(Regex, String) + +/// re.compile; panics on a bad pattern, since every pattern here is a literal. +pub fn compile(pattern: String, flags: List(Flag)) -> Regex { + let assert Ok(r) = compile_ffi(pattern, flags) + r +} + +/// bool(r.search(s)) +@external(erlang, "fabric_ffi", "regex_check") +pub fn search(r: Regex, s: String) -> Bool + +/// Every match as [whole, group 1, ...]; a group that did not take part is "". +@external(erlang, "fabric_ffi", "regex_scan") +pub fn scan(r: Regex, s: String) -> List(List(String)) + +/// r.search(s): the first match's [whole, group 1, ...]. +pub fn first(r: Regex, s: String) -> Result(List(String), Nil) { + list.first(scan(r, s)) +} + +/// r.findall(s) for a pattern with one group, or no group. +pub fn findall(r: Regex, s: String) -> List(String) { + list.map(scan(r, s), fn(m) { + case m { + [_, g, ..] -> g + [w] -> w + [] -> "" + } + }) +} + +@external(erlang, "fabric_ffi", "regex_replace") +fn replace_ffi(r: Regex, s: String, with: String, all: Bool) -> String + +/// r.sub(with, s): `\1` in `with` is group 1 and `&` is the whole match (Erlang's +/// re:replace syntax; write `\&` for a literal &). +pub fn sub(r: Regex, s: String, with: String) -> String { + replace_ffi(r, s, with, True) +} + +/// r.sub(with, s, count=1) +pub fn sub_first(r: Regex, s: String, with: String) -> String { + replace_ffi(r, s, with, False) +} + +/// r.split(s), for a pattern without groups. +@external(erlang, "fabric_ffi", "regex_split") +pub fn split(r: Regex, s: String) -> List(String) diff --git a/tools/src/fabric/refs_check.gleam b/tools/src/fabric/refs_check.gleam new file mode 100644 index 0000000..d3f5bac --- /dev/null +++ b/tools/src/fabric/refs_check.gleam @@ -0,0 +1,169 @@ +//// refs-check: every reference a paper cites must have been fetched and logged. +//// +//// For each papers//refs.bib, the REFERENCES.md next to it must have one table row per +//// BibTeX key, with status verified or partial, and no row for a key that refs.bib lacks. Every +//// entry must carry a doi or url, so a reader can follow it. An `unverified` row fails the +//// check: an agent that could not fetch a source must not cite it (AGENTS.md). +//// +//// tools/fabric refs-check papers/*/refs.bib + +import fabric/pystr +import fabric/re +import fabric/sys +import gleam/int +import gleam/list +import gleam/string + +/// The usage, printed as refs_check.py printed its __doc__. +const doc = + "refs_check — every reference a paper cites must have been fetched and logged. + +For each papers//refs.bib, the REFERENCES.md next to it must have one table row per BibTeX +key, with status verified or partial, and no row for a key that refs.bib lacks. Every entry must +carry a doi or url, so a reader can follow it. An `unverified` row fails the check: an agent that +could not fetch a source must not cite it (AGENTS.md). + + tools/fabric refs-check papers/*/refs.bib +" + +const ok = ["verified", "partial"] + +/// dict(pairs): a key keeps its first place and takes its last value. +fn to_dict(pairs: List(#(String, String))) -> List(#(String, String)) { + list.fold(pairs, [], fn(acc, pair) { + case list.key_find(acc, pair.0) { + Ok(_) -> list.key_set(acc, pair.0, pair.1) + Error(_) -> [pair, ..acc] + } + }) + |> list.reverse +} + +/// The BibTeX entries, key to body, @comment left out. +pub fn entries(bib: String) -> List(#(String, String)) { + re.compile("@(\\w+)\\s*\\{\\s*([^,\\s]+)\\s*,(.*?)\\n\\}", [re.Dotall]) + |> re.scan(bib) + |> list.filter_map(fn(m) { + case m { + [_, kind, key, body] -> + case string.lowercase(kind) { + "comment" -> Error(Nil) + _ -> Ok(#(key, body)) + } + _ -> Error(Nil) + } + }) + |> to_dict +} + +/// REFERENCES.md's rows, key to status. +pub fn rows(log: String) -> List(#(String, String)) { + re.compile( + "^\\|\\s*([A-Za-z0-9_:-]+)\\s*\\|\\s*(verified|partial|unverified)\\s*\\|", + [re.Multiline], + ) + |> re.scan(log) + |> list.filter_map(fn(m) { + case m { + [_, key, status] -> Ok(#(key, status)) + _ -> Error(Nil) + } + }) + |> to_dict +} + +/// The errors for one refs.bib (named `bib_path`, as printed), given its text and its +/// REFERENCES.md's text, or Error when there is none. +pub fn check_texts( + bib_path: String, + bib: String, + log_path: String, + log: Result(String, Nil), +) -> List(String) { + case log { + Error(_) -> [bib_path <> ": no REFERENCES.md next to it"] + Ok(log) -> { + let bib = entries(bib) + let rows = rows(log) + let link = re.compile("^\\s*(doi|url)\\s*=", [re.Multiline]) + let per_key = + list.flat_map(bib, fn(entry) { + let #(key, body) = entry + let row = case list.key_find(rows, key) { + Error(_) -> [ + bib_path <> ": " <> key <> " has no row in REFERENCES.md", + ] + Ok(status) -> + case list.contains(ok, status) { + True -> [] + False -> [ + bib_path + <> ": " + <> key + <> " is " + <> status + <> "; fetch and check it, or remove it", + ] + } + } + let url = case re.search(link, body) { + True -> [] + False -> [bib_path <> ": " <> key <> " has neither doi nor url"] + } + list.append(row, url) + }) + let extra = + list.filter_map(rows, fn(row) { + case list.key_find(bib, row.0) { + Ok(_) -> Error(Nil) + Error(_) -> + Ok(log_path <> ": row " <> row.0 <> " is not in refs.bib") + } + }) + list.append(per_key, extra) + } + } +} + +fn check(path: String) -> List(String) { + let shown = pystr.path_str(path) + let log_path = pystr.path_str(sys.join(sys.parent(shown), "REFERENCES.md")) + case pystr.read_text(path) { + Error(e) -> [e] + Ok(bib) -> { + let log = case sys.exists(log_path) { + False -> Error(Nil) + True -> + case pystr.read_text(log_path) { + Ok(t) -> Ok(t) + Error(_) -> Error(Nil) + } + } + check_texts(shown, bib, log_path, log) + } + } +} + +pub fn main(args: List(String)) -> Int { + case args { + [] -> { + sys.eprintln(doc) + 2 + } + _ -> { + let errs = list.flat_map(args, check) + list.each(errs, fn(e) { sys.eprintln("refs_check: " <> e) }) + case errs { + [] -> { + sys.eprintln( + "refs_check: " + <> int.to_string(list.length(args)) + <> " bibliography file(s) ok", + ) + 0 + } + _ -> 1 + } + } + } +} diff --git a/tools/src/fabric/ste_check.gleam b/tools/src/fabric/ste_check.gleam new file mode 100644 index 0000000..999831f --- /dev/null +++ b/tools/src/fabric/ste_check.gleam @@ -0,0 +1,340 @@ +//// ste-check: the ASD-STE100 text of a paper keeps what the original says. +//// +//// An STE pass may change words, never facts. This compares papers//prose-ste/ with +//// papers//prose/, file by file, and fails when the citation markers (as a multiset), the +//// anchors, the quotations, the numbers or the [ANALYSIS] and [AUTHOR ...] markers differ. It +//// reports, without failing, sentences over 25 words outside quotations, for the STE100.md log. +//// +//// With --translation it compares papers//prose-pt-BR/ instead: numbers are compared +//// without their separators (3,5 is 3.5), markers are read in Portuguese, and a quotation may +//// gain a translation but never lose its English original. + +import fabric/re +import fabric/sys +import gleam/dict.{type Dict} +import gleam/int +import gleam/list +import gleam/option +import gleam/result +import gleam/string + +const long = 25 + +const usage = + "ste_check — the ASD-STE100 text of a paper keeps what the original says. + +An STE pass may change words, never facts. This check compares papers//prose-ste/ with +papers//prose/, file by file, and fails when any of these differ: the citation markers (as +a multiset), the cross-reference and heading anchors, the quotations, the numbers, and the +[ANALYSIS] and [AUTHOR ...] markers. It also reports, without failing, sentences over 25 words +outside quotations, so the STE100.md log can list them. + +With --translation it compares papers//prose-pt-BR/ instead, the Brazilian Portuguese +edition: the same facts, except that numbers are compared without their separators (3,5 is 3.5), +the markers are read in Portuguese, and a quotation may gain a translation but never lose its +English original. + + tools/fabric ste-check closed-source + tools/fabric ste-check --translation closed-source +" + +/// A multiset of strings, as Python's Counter. +pub type Counter = + Dict(String, Int) + +pub fn counter(items: List(String)) -> Counter { + list.fold(items, dict.new(), fn(d, x) { + dict.upsert(d, x, fn(n) { + case n { + option.Some(n) -> n + 1 + option.None -> 1 + } + }) + }) +} + +/// Counter a - b: only positive counts stay. +fn minus(a: Counter, b: Counter) -> Counter { + dict.fold(a, dict.new(), fn(acc, k, n) { + let m = n - result.unwrap(dict.get(b, k), 0) + case m > 0 { + True -> dict.insert(acc, k, m) + False -> acc + } + }) +} + +/// sorted(c.elements()) +pub fn elements(c: Counter) -> List(String) { + dict.to_list(c) + |> list.flat_map(fn(p) { list.repeat(p.0, p.1) }) + |> list.sort(string.compare) +} + +/// str.split() with no argument: Python's whitespace, empty pieces dropped. +fn words(s: String) -> List(String) { + re.split(re.compile("[\\s\\x{1c}-\\x{1f}]+", []), s) + |> list.filter(fn(w) { w != "" }) +} + +fn strip_code_urls(text: String) -> String { + let t = re.sub(re.compile("`[^`]*`", []), text, " ") + let t = re.sub(re.compile("]+>|https?://\\S+", []), t, " ") + re.sub(re.compile("^---\\n.*?\\n---\\n", [re.Dotall]), t, "") +} + +fn cite_re() { + re.compile("-?@[A-Za-z][\\w:-]*", []) +} + +fn quote_re() { + re.compile("[\"“]([^\"”]{3,})[\"”]", []) +} + +const pt_markers = [ + #("ANÁLISE", "ANALYSIS"), + #("RELATO DO AUTOR", "AUTHOR ANECDOTE"), + #("AUTORES A CONFIRMAR", "AUTHORS TO CONFIRM"), +] + +fn strip_separators(n: String) -> String { + re.sub(re.compile("[.,]", []), n, "") +} + +/// The facts a pass must keep, by kind, in the order the errors are reported. +pub fn facts(text: String, pt: Bool) -> List(#(String, Counter)) { + let t = strip_code_urls(text) + let numbers = + re.findall( + re.compile( + "(? list.map(rstrip_dots) + let markers = case pt { + True -> + re.findall( + re.compile( + "\\*\\*\\[(" + <> string.join(list.map(pt_markers, fn(p) { p.0 }), "|") + <> ")", + [], + ), + t, + ) + |> list.map(fn(m) { result.unwrap(list.key_find(pt_markers, m), m) }) + False -> + re.findall( + re.compile("\\*\\*\\[(ANALYSIS|AUTHOR ANECDOTE|AUTHORS TO CONFIRM)", []), + t, + ) + } + [ + #("citations", counter(re.findall(cite_re(), t))), + #("anchors", counter(re.findall(re.compile("\\{[#.][^}]*\\}", []), t))), + #( + "quotations", + counter( + list.map(re.findall(quote_re(), t), fn(q) { string.join(words(q), " ") }), + ), + ), + #("numbers", case pt { + True -> counter(list.map(numbers, strip_separators)) + False -> counter(numbers) + }), + #("markers", counter(markers)), + ] +} + +/// n.rstrip(".,") +fn rstrip_dots(n: String) -> String { + case string.ends_with(n, ".") || string.ends_with(n, ",") { + True -> rstrip_dots(string.drop_end(n, 1)) + False -> n + } +} + +pub fn long_sentences(text: String) -> List(String) { + let t = strip_code_urls(text) + let t = re.sub(re.compile("^#.*$|^\\|.*$|^:::.*$", [re.Multiline]), t, " ") + let t = re.sub(re.compile("\\[@[^\\]]*\\]|\\[-@[^\\]]*\\]", []), t, " ") + let t = re.sub(quote_re(), t, "Q") + let has_word = re.compile("\\w", []) + re.split( + re.compile("(?<=[.!?])\\s+(?=[A-Z*\\[(])", []), + string.join(words(t), " "), + ) + |> list.filter_map(fn(s) { + let n = list.count(words(s), fn(w) { re.search(has_word, w) }) + case n > long { + True -> { + let head = + string.to_utf_codepoints(s) + |> list.take(90) + |> string.from_utf_codepoints + Ok(int.to_string(n) <> " words: " <> head <> "…") + } + False -> Error(Nil) + } + }) +} + +pub fn compare(orig: String, ste: String, translation: Bool) -> List(String) { + let a = facts(orig, False) + let b = facts(ste, translation) + list.flat_map(a, fn(pair) { + let #(kind, ca) = pair + let ca = case kind, translation { + "numbers", True -> counter(list.map(elements(ca), strip_separators)) + _, _ -> ca + } + let cb = result.unwrap(list.key_find(b, kind), dict.new()) + let lost = minus(ca, cb) + let added = minus(cb, ca) + // Splitting an [ANALYSIS] paragraph in two (the six-sentence rule) repeats its marker; + // that marks the same interpretation, so an added ANALYSIS marker is not a change. + let added = case kind { + "markers" -> dict.delete(added, "ANALYSIS") + _ -> added + } + // A translation may follow an English quotation with its Portuguese version. + let added = case kind, translation { + "quotations", True -> dict.new() + _, _ -> added + } + list.flatten([ + report(kind, "lost", lost), + report(kind, "added", added), + ]) + }) +} + +fn report(kind: String, what: String, c: Counter) -> List(String) { + case dict.is_empty(c) { + True -> [] + False -> [kind <> " " <> what <> ": " <> repr_list(elements(c))] + } +} + +/// Python's repr() of a list of str. +pub fn repr_list(items: List(String)) -> String { + "[" <> string.join(list.map(items, repr), ", ") <> "]" +} + +/// Python's repr() of a str. +pub fn repr(s: String) -> String { + let q = case string.contains(s, "'") && !string.contains(s, "\"") { + True -> "\"" + False -> "'" + } + let body = + string.to_utf_codepoints(s) + |> list.map(fn(cp) { + case string.utf_codepoint_to_int(cp) { + 0x5c -> "\\\\" + 0x0a -> "\\n" + 0x0d -> "\\r" + 0x09 -> "\\t" + 0x27 if q == "'" -> "\\'" + c if c < 0x20 || c == 0x7f -> "\\x" <> hex2(c) + _ -> string.from_utf_codepoints([cp]) + } + }) + |> string.concat + q <> body <> q +} + +fn hex2(c: Int) -> String { + string.lowercase(string.pad_start(int.to_base16(c), 2, "0")) +} + +fn read(path: String) -> String { + let assert Ok(text) = sys.read(path) + text +} + +pub fn check(slug: String, translation: Bool) -> Int { + let base = "papers/" <> slug + let ste_dir = case translation { + True -> base <> "/prose-pt-BR" + False -> base <> "/prose-ste" + } + let #(errs, longs) = + sys.glob(base <> "/prose", ".md") + |> list.fold(#([], []), fn(acc, o) { + let #(errs, longs) = acc + let name = sys.base_name(o) + let s = sys.join(ste_dir, name) + case sys.exists(s) { + False -> #([s <> " is missing", ..errs], longs) + True -> { + let ste = read(s) + let errs = + list.fold(compare(read(o), ste, translation), errs, fn(e, x) { + [name <> ": " <> x, ..e] + }) + let longs = case translation { + True -> longs + False -> + list.fold(long_sentences(ste), longs, fn(l, x) { + [name <> ": " <> x, ..l] + }) + } + #(errs, longs) + } + } + }) + let errs = list.reverse(errs) + let longs = list.reverse(longs) + let meta = ste_dir <> "/00-metadata.md" + let longs = case sys.exists(meta) && !translation { + True -> + list.append( + longs, + long_sentences(string.replace(read(meta), "abstract: |", "")) + |> list.map(fn(l) { "00-metadata.md: " <> l }), + ) + False -> longs + } + list.each(longs, fn(l) { sys.eprintln("ste_check: note: " <> l) }) + list.each(errs, fn(e) { sys.eprintln("ste_check: " <> e) }) + case errs, translation { + [], True -> + sys.eprintln( + "ste_check: " + <> slug + <> " pt-BR: citations, anchors, quotations, numbers and markers kept", + ) + [], False -> + sys.eprintln( + "ste_check: " + <> slug + <> ": citations, anchors, quotations, numbers and markers unchanged; " + <> int.to_string(list.length(longs)) + <> " sentence(s) over " + <> int.to_string(long) + <> " words", + ) + _, _ -> Nil + } + case errs { + [] -> 0 + _ -> 1 + } +} + +pub fn main(args: List(String)) -> Int { + let #(translation, rest) = case args { + ["--translation", ..rest] -> #(True, rest) + _ -> #(False, args) + } + case rest { + [slug] -> check(slug, translation) + _ -> { + sys.eprintln(usage) + 2 + } + } +} diff --git a/tools/src/fabric/sys.gleam b/tools/src/fabric/sys.gleam new file mode 100644 index 0000000..2a55db1 --- /dev/null +++ b/tools/src/fabric/sys.gleam @@ -0,0 +1,214 @@ +//// What the scripts needed from Python's os, sys, subprocess, pathlib, tempfile, hashlib, tarfile +//// and urllib, over fabric_ffi.erl. Paths are relative to the repository root unless absolute: +//// the `tools/fabric` wrapper starts the VM there, as `ROOT` was in each script. + +import gleam/bit_array +import gleam/int +import gleam/list +import gleam/string + +@external(erlang, "fabric_ffi", "args") +pub fn args() -> List(String) + +@external(erlang, "fabric_ffi", "getenv") +pub fn getenv(name: String) -> Result(String, Nil) + +/// Ends the program with this exit status, after flushing output. +@external(erlang, "fabric_ffi", "halt") +pub fn exit(code: Int) -> a + +@external(erlang, "fabric_ffi", "eprint") +fn eprint(text: String) -> Nil + +/// print(..., file=sys.stderr) +pub fn eprintln(text: String) -> Nil { + eprint(text <> "\n") +} + +@external(erlang, "fabric_ffi", "read_file") +fn read_bits(path: String) -> Result(BitArray, String) + +pub fn read_bytes(path: String) -> Result(BitArray, String) { + read_bits(path) +} + +/// Path.read_text(); invalid UTF-8 is an error. +pub fn read(path: String) -> Result(String, String) { + case read_bits(path) { + Ok(b) -> + case bit_array.to_string(b) { + Ok(s) -> Ok(s) + Error(_) -> Error(path <> ": not UTF-8") + } + Error(e) -> Error(e) + } +} + +@external(erlang, "fabric_ffi", "write_file") +fn write_bits(path: String, data: BitArray) -> Result(Nil, String) + +pub fn write(path: String, text: String) -> Result(Nil, String) { + write_bits(path, <>) +} + +pub fn write_bytes(path: String, data: BitArray) -> Result(Nil, String) { + write_bits(path, data) +} + +@external(erlang, "fabric_ffi", "exists") +pub fn exists(path: String) -> Bool + +@external(erlang, "fabric_ffi", "is_dir") +pub fn is_dir(path: String) -> Bool + +/// Names in a directory, sorted (byte order, as Python's sorted() on str for ASCII names). +@external(erlang, "fabric_ffi", "list_dir") +pub fn list_dir(path: String) -> Result(List(String), String) + +/// sorted(Path(dir).glob("*" <> suffix)) for a plain suffix such as ".md", as full paths. +pub fn glob(dir: String, suffix: String) -> List(String) { + case list_dir(dir) { + Ok(names) -> + names + |> list.filter(fn(n) { string.ends_with(n, suffix) }) + |> list.map(fn(n) { join(dir, n) }) + Error(_) -> [] + } +} + +/// mkdir(parents=True, exist_ok=True) +pub fn make_dirs(path: String) -> Result(Nil, String) { + make_dirs_ffi(path) +} + +@external(erlang, "fabric_ffi", "make_dirs") +fn make_dirs_ffi(path: String) -> Result(Nil, String) + +@external(erlang, "fabric_ffi", "temp_dir") +pub fn temp_dir() -> String + +/// shutil.rmtree, ignoring errors. +@external(erlang, "fabric_ffi", "remove_all") +pub fn remove_all(path: String) -> Nil + +pub fn join(a: String, b: String) -> String { + case a, string.starts_with(b, "/") { + _, True -> b + "", _ -> b + _, _ -> + case string.ends_with(a, "/") { + True -> a <> b + False -> a <> "/" <> b + } + } +} + +/// Path(p).name +pub fn base_name(path: String) -> String { + case list.last(string.split(path, "/")) { + Ok(n) -> n + Error(_) -> path + } +} + +/// Path(p).parent, as a string ("." for a bare name). +pub fn parent(path: String) -> String { + case string.split(path, "/") |> list.reverse { + [_] -> "." + [_, ..rest] -> + case list.reverse(rest) |> string.join("/") { + "" -> "/" + p -> p + } + [] -> "." + } +} + +/// Path(p).stem +pub fn stem(path: String) -> String { + let name = base_name(path) + case string.split(name, ".") |> list.reverse { + [_, first, ..rest] if first != "" || rest != [] -> + list.reverse([first, ..rest]) |> string.join(".") + _ -> name + } +} + +pub type Run { + Run(status: Int, stdout: String) +} + +@external(erlang, "fabric_ffi", "run") +fn run_ffi( + command: String, + args: List(String), + cwd: String, +) -> Result(#(Int, BitArray), String) + +/// subprocess.run(capture_output=True): stdout is captured, stderr passes through. +/// Error only when the command cannot be started. +pub fn run( + command: String, + args: List(String), + cwd: String, +) -> Result(Run, String) { + case run_ffi(command, args, cwd) { + Ok(#(status, out)) -> + case bit_array.to_string(out) { + Ok(s) -> Ok(Run(status, s)) + Error(_) -> Error(command <> ": output is not UTF-8") + } + Error(e) -> Error(e) + } +} + +/// subprocess.run(check=True, ...).stdout; a non-zero status is an error. +pub fn check_output( + command: String, + args: List(String), + cwd: String, +) -> Result(String, String) { + case run(command, args, cwd) { + Ok(Run(0, out)) -> Ok(out) + Ok(Run(status, _)) -> + Error( + "Command '" + <> string.join([command, ..args], " ") + <> "' returned non-zero exit status " + <> int.to_string(status) + <> ".", + ) + Error(e) -> Error(e) + } +} + +@external(erlang, "fabric_ffi", "sha256_hex") +pub fn sha256_hex(data: BitArray) -> String + +@external(erlang, "fabric_ffi", "http") +fn http_ffi( + method: String, + url: String, + headers: List(#(String, String)), + body: BitArray, +) -> Result(#(Int, BitArray), String) + +/// One HTTP(S) request; returns the status and the body. +pub fn http( + method: String, + url: String, + headers: List(#(String, String)), + body: BitArray, +) -> Result(#(Int, BitArray), String) { + http_ffi(method, url, headers, body) +} + +/// A gzipped tarball of #(name in the archive, path on disk). +@external(erlang, "fabric_ffi", "tar_gz_create") +pub fn tar_gz_create( + tarball: String, + files: List(#(String, String)), +) -> Result(Nil, String) + +@external(erlang, "fabric_ffi", "tar_gz_extract") +pub fn tar_gz_extract(tarball: String, dir: String) -> Result(Nil, String) diff --git a/tools/src/fabric_ffi.erl b/tools/src/fabric_ffi.erl new file mode 100644 index 0000000..c2a12ef --- /dev/null +++ b/tools/src/fabric_ffi.erl @@ -0,0 +1,198 @@ +-module(fabric_ffi). +-export([args/0, getenv/1, halt/1, eprint/1, read_file/1, write_file/2, exists/1, is_dir/1, + list_dir/1, make_dirs/1, temp_dir/0, remove_all/1, run/3, sha256_hex/1, + regex_compile/2, regex_check/2, regex_scan/2, regex_replace/4, regex_split/2, + json_decode/1, http/4, tar_gz_create/2, tar_gz_extract/2]). + +args() -> [unicode:characters_to_binary(A) || A <- init:get_plain_arguments()]. + +getenv(Name) -> + case os:getenv(binary_to_list(Name)) of + false -> {error, nil}; + V -> {ok, unicode:characters_to_binary(V)} + end. + +halt(Code) -> erlang:halt(Code, [{flush, true}]). + +eprint(S) -> io:put_chars(standard_error, S). + +read_file(P) -> + case file:read_file(P) of + {ok, B} -> {ok, B}; + {error, R} -> {error, reason(P, R)} + end. + +write_file(P, B) -> + case file:write_file(P, B) of + ok -> {ok, nil}; + {error, R} -> {error, reason(P, R)} + end. + +exists(P) -> filelib:is_file(P). +is_dir(P) -> filelib:is_dir(P). + +list_dir(P) -> + case file:list_dir(P) of + {ok, Names} -> {ok, lists:sort([unicode:characters_to_binary(N) || N <- Names])}; + {error, R} -> {error, reason(P, R)} + end. + +make_dirs(P) -> + case filelib:ensure_path(P) of + ok -> {ok, nil}; + {error, R} -> {error, reason(P, R)} + end. + +temp_dir() -> + Base = case os:getenv("TMPDIR") of false -> "/tmp"; T -> T end, + D = filename:join(Base, "fabric-" ++ integer_to_list(erlang:unique_integer([positive])) + ++ "-" ++ os:getpid()), + ok = file:make_dir(D), + unicode:characters_to_binary(D). + +remove_all(P) -> _ = file:del_dir_r(P), nil. + +reason(P, R) -> unicode:characters_to_binary([P, ": ", file:format_error(R)]). + +%% run(Command, Args, Cwd): stdout is captured, stderr goes to ours, as Python's +%% subprocess.run(capture_output=True) with stderr left alone. Cwd "" keeps ours. +run(Cmd, Args, Cwd) -> + case os:find_executable(binary_to_list(Cmd)) of + false -> {error, <>}; + Exe -> + Opts = [{args, [binary_to_list(A) || A <- Args]}, exit_status, binary, use_stdio, + hide | case Cwd of <<>> -> []; _ -> [{cd, binary_to_list(Cwd)}] end], + Port = open_port({spawn_executable, Exe}, Opts), + collect(Port, []) + end. + +collect(Port, Acc) -> + receive + {Port, {data, D}} -> collect(Port, [Acc, D]); + {Port, {exit_status, S}} -> {ok, {S, iolist_to_binary(Acc)}} + end. + +sha256_hex(B) -> binary:encode_hex(crypto:hash(sha256, B), lowercase). + +regex_compile(Pattern, Opts) -> + %% Gleam sends caseless, multiline and dotall, which are re option names already. + case re:compile(Pattern, [unicode, ucp | Opts]) of + {ok, R} -> {ok, {R, groups(unicode:characters_to_list(Pattern), 0)}}; + {error, {Msg, At}} -> {error, unicode:characters_to_binary(io_lib:format("~s at ~p", [Msg, At]))} + end. + +%% Erlang's re drops unset groups at the end of a match, so the group count is kept with the +%% pattern and every group is asked for by number. +groups([], N) -> N; +groups([$\\, _ | T], N) -> groups(T, N); +groups([$[ | T], N) -> groups(class(T), N); +groups([$(, $?, $< | [C | _] = T], N) when C =/= $=, C =/= $! -> groups(T, N + 1); +groups([$(, $?, $P, $< | T], N) -> groups(T, N + 1); +groups([$(, $? | T], N) -> groups(T, N); +groups([$( | T], N) -> groups(T, N + 1); +groups([_ | T], N) -> groups(T, N). + +class([$] | T]) -> skip_class(T); +class([$^, $] | T]) -> skip_class(T); +class(T) -> skip_class(T). +skip_class([]) -> []; +skip_class([$\\, _ | T]) -> skip_class(T); +skip_class([$] | T]) -> T; +skip_class([_ | T]) -> skip_class(T). + +regex_check({R, _}, S) -> re:run(S, R, [{capture, none}]) =:= match. + +%% Every match, as [whole, group1, ...]; a group that took no part is "". +regex_scan({R, N}, S) -> + case re:run(S, R, [global, {capture, lists:seq(0, N), binary}]) of + {match, Ms} -> Ms; + nomatch -> [] + end. + +regex_replace({R, _}, S, With, All) -> + Opts = [{return, binary} | case All of true -> [global]; false -> [] end], + re:replace(S, R, With, Opts). + +regex_split({R, _}, S) -> re:split(S, R, [{return, binary}]). + +%% JSON with object key order kept, as Python's dict keeps it. +json_decode(B) -> + try + {V, ok, <<>>} = json:decode(B, ok, #{ + object_start => fun(_) -> [] end, + object_push => fun(K, V, Acc) -> [{K, V} | Acc] end, + object_finish => fun(Acc, Old) -> {{object, lists:reverse(Acc)}, Old} end, + array_start => fun(_) -> [] end, + array_push => fun(V, Acc) -> [V | Acc] end, + array_finish => fun(Acc, Old) -> {{array, lists:reverse(Acc)}, Old} end, + null => null}), + {ok, wrap(V)} + catch + _:E -> {error, unicode:characters_to_binary(io_lib:format("~p", [E]))} + end. + +wrap(null) -> null; +wrap(true) -> {bool, true}; +wrap(false) -> {bool, false}; +wrap(I) when is_integer(I) -> {int, I}; +wrap(F) when is_float(F) -> {float, F}; +wrap(S) when is_binary(S) -> {string, S}; +wrap({array, L}) -> {array, [wrap(X) || X <- L]}; +wrap({object, KVs}) -> {object, [{K, wrap(V)} || {K, V} <- KVs]}. + +%% http(Method, Url, Headers, Body) -> {ok, {Status, Body}}. HTTPS_PROXY and SSL_CERT_FILE are +%% honoured, since httpc reads neither on its own. +http(Method, Url, Headers, Body) -> + {ok, _} = application:ensure_all_started([inets, ssl]), + proxy(), + Hs = [{binary_to_list(K), binary_to_list(V)} || {K, V} <- Headers], + Ssl = [{ssl, [{verify, verify_peer}, cacerts(), + {customize_hostname_check, [{match_fun, public_key:pkix_verify_hostname_match_fun(https)}]}]}, + {timeout, 120000}], + M = list_to_atom(string:lowercase(binary_to_list(Method))), + %% A request without a body and without a Content-Type sends neither, as urllib does. + Req = case {M, Body, lists:keymember("Content-Type", 1, Hs)} of + {get, _, _} -> {binary_to_list(Url), Hs}; + {delete, _, _} -> {binary_to_list(Url), Hs}; + {_, <<>>, false} -> {binary_to_list(Url), Hs, "", Body}; + _ -> {binary_to_list(Url), Hs, content_type(Hs), Body} + end, + case httpc:request(M, Req, Ssl, [{body_format, binary}]) of + {ok, {{_, Status, _}, _, RespBody}} -> {ok, {Status, RespBody}}; + {error, R} -> {error, unicode:characters_to_binary(io_lib:format("~p", [R]))} + end. + +content_type(Hs) -> + case lists:keyfind("Content-Type", 1, Hs) of + {_, T} -> T; + false -> "application/octet-stream" + end. + +cacerts() -> + case os:getenv("SSL_CERT_FILE") of + false -> {cacerts, public_key:cacerts_get()}; + F -> {cacertfile, F} + end. + +proxy() -> + case os:getenv("HTTPS_PROXY", os:getenv("https_proxy", "")) of + "" -> ok; + P -> + #{host := H, port := Port} = uri_string:parse(P), + NoProxy = string:split(os:getenv("NO_PROXY", os:getenv("no_proxy", "")), ",", all), + ok = httpc:set_options([{https_proxy, {{H, Port}, NoProxy}}, {proxy, {{H, Port}, NoProxy}}]) + end. + +%% tar_gz_create(Tarball, [{NameInArchive, Path}]) +tar_gz_create(Tarball, Files) -> + case erl_tar:create(binary_to_list(Tarball), + [{binary_to_list(N), binary_to_list(P)} || {N, P} <- Files], [compressed]) of + ok -> {ok, nil}; + {error, R} -> {error, unicode:characters_to_binary(erl_tar:format_error(R))} + end. + +tar_gz_extract(Tarball, Dir) -> + case erl_tar:extract(binary_to_list(Tarball), [compressed, {cwd, binary_to_list(Dir)}]) of + ok -> {ok, nil}; + {error, R} -> {error, unicode:characters_to_binary(erl_tar:format_error(R))} + end. diff --git a/tools/test/fabric_test.gleam b/tools/test/fabric_test.gleam new file mode 100644 index 0000000..902c4da --- /dev/null +++ b/tools/test/fabric_test.gleam @@ -0,0 +1,5 @@ +import gleeunit + +pub fn main() -> Nil { + gleeunit.main() +} diff --git a/tools/test/pystr_test.gleam b/tools/test/pystr_test.gleam new file mode 100644 index 0000000..f39e2bf --- /dev/null +++ b/tools/test/pystr_test.gleam @@ -0,0 +1,28 @@ +import fabric/pystr + +pub fn splitlines_test() { + assert pystr.splitlines("") == [] + assert pystr.splitlines("a\n\nb\r\nc\n") == ["a", "", "b", "c"] + assert pystr.splitlines("a\u{2028}b") == ["a", "b"] +} + +pub fn strip_test() { + assert pystr.strip(" \t a b \n") == "a b" +} + +pub fn repr_test() { + assert pystr.repr("x") == "'x'" + assert pystr.repr("it's") == "\"it's\"" + assert pystr.repr("'\"") == "'\\'\"'" + assert pystr.repr("a\\b\u{01}\u{a0}é") == "'a\\\\b\\x01\\xa0é'" +} + +pub fn path_str_test() { + assert pystr.path_str("./a//b/./c") == "a/b/c" + assert pystr.path_str("./") == "." + assert pystr.path_str("/a/b") == "/a/b" +} + +pub fn duplicates_test() { + assert pystr.duplicates(["b", "a", "b", "c", "a"]) == ["a", "b"] +} diff --git a/tools/test/refs_check_test.gleam b/tools/test/refs_check_test.gleam new file mode 100644 index 0000000..9e9c9b8 --- /dev/null +++ b/tools/test/refs_check_test.gleam @@ -0,0 +1,41 @@ +import fabric/refs_check + +const bib = + "@misc{a,\n url = {https://x}\n}\n@misc{b,\n title = {t}\n}\n@comment{c,\n}\n" + +pub fn entries_test() { + assert refs_check.entries(bib) + == [#("a", "\n url = {https://x}"), #("b", "\n title = {t}")] +} + +pub fn check_test() { + let log = + "| Key | Status |\n|---|---|\n| a | verified | x |\n| c | partial | y |\n" + assert refs_check.check_texts("refs.bib", bib, "REFERENCES.md", Ok(log)) + == [ + "refs.bib: b has no row in REFERENCES.md", + "refs.bib: b has neither doi nor url", + "REFERENCES.md: row c is not in refs.bib", + ] + assert refs_check.check_texts( + "refs.bib", + "@misc{a,\n doi = {x}\n}", + "REFERENCES.md", + Ok("| a | unverified | x |\n"), + ) + == ["refs.bib: a is unverified; fetch and check it, or remove it"] + assert refs_check.check_texts( + "p/refs.bib", + bib, + "p/REFERENCES.md", + Error(Nil), + ) + == ["p/refs.bib: no REFERENCES.md next to it"] +} + +pub fn rows_keep_first_place_last_value_test() { + assert refs_check.rows( + "| a | partial |\n| b | verified |\n| a | unverified |\n", + ) + == [#("a", "unverified"), #("b", "verified")] +} diff --git a/tools/test/ste_check_test.gleam b/tools/test/ste_check_test.gleam new file mode 100644 index 0000000..3f06f41 --- /dev/null +++ b/tools/test/ste_check_test.gleam @@ -0,0 +1,92 @@ +import fabric/ste_check.{compare, long_sentences} +import gleam/list +import gleam/string + +const o = + "A claim [@a; @b]. He said \"it is so\" in 1991. **[ANALYSIS]** See Section [-@sec:x]." + +pub fn same_facts_in_other_words_test() { + assert compare( + o, + "A claim is true [@a; @b]. In 1991 he said \"it is so\". **[ANALYSIS]** See Section [-@sec:x].", + False, + ) + == [] +} + +pub fn lost_and_added_facts_test() { + let errs = + compare( + o, + "A claim [@a]. He said \"it is\" in 1992. See Section [-@sec:x].", + False, + ) + assert errs + == [ + "citations lost: ['@b']", + "quotations lost: ['it is so']", + "quotations added: ['it is']", + "numbers lost: ['1991']", + "numbers added: ['1992']", + "markers lost: ['ANALYSIS']", + ] +} + +pub fn split_analysis_paragraph_test() { + assert compare( + "**[ANALYSIS]** a. b.", + "**[ANALYSIS]** a.\n\n**[ANALYSIS]** b.", + False, + ) + == [] +} + +pub fn long_sentences_test() { + let assert [note] = + long_sentences(string.join(list.repeat("word", 30), " ") <> ".") + assert string.starts_with(note, "30 words: word word") + assert string.ends_with(note, "…") + assert long_sentences("Short one.") == [] +} + +pub fn translation_keeps_facts_test() { + let pt = + "Uma tese [@a; @b]. Ele disse \"it is so\" (em tradução livre: \"é assim\") em 1991. **[ANÁLISE]** Ver a Seção [-@sec:x]." + assert compare(o, pt, True) == [] + assert compare("Cost 3.5 and 1,000.", "Custo 3,5 e 1.000.", True) == [] +} + +pub fn translation_loses_facts_test() { + let errs = + compare( + o, + "Uma tese [@a]. Ele disse \"é assim\" em 1991. Ver a Seção [-@sec:x].", + True, + ) + assert errs + == [ + "citations lost: ['@b']", + "quotations lost: ['it is so']", + "markers lost: ['ANALYSIS']", + ] +} + +pub fn front_matter_code_and_urls_are_skipped_test() { + assert compare( + "---\nx: 1\n---\nA `9` https://c.d/3 b.", + "A b.", + False, + ) + == [] +} + +pub fn repr_as_python_test() { + assert ste_check.repr("it's") == "\"it's\"" + assert ste_check.repr("a'b\"c\\") == "'a\\'b\"c\\\\'" + assert ste_check.repr("é\t\u{01}") == "'é\\t\\x01'" +} + +pub fn counter_elements_sorted_test() { + assert ste_check.elements(ste_check.counter(["b", "a", "b", "é"])) + == ["a", "b", "b", "é"] +}