diff --git a/.github/workflows/issues.yml b/.github/workflows/issues.yml new file mode 100644 index 0000000..b17575d --- /dev/null +++ b/.github/workflows/issues.yml @@ -0,0 +1,31 @@ +# Keeps GitHub issues in step with issues/*.md: on every push to main (merges +# included) that touches the folder, scripts/sync-issues.sh creates, edits or +# closes the matching issues. Edit the files, not the issues: the next sync +# overwrites an issue's title and body. +name: Sync issues + +on: + push: + branches: [main] + paths: ['issues/**', 'scripts/sync-issues.sh'] + workflow_dispatch: + +permissions: + contents: read + issues: write + +# One sync at a time, so two pushes never create the same issue twice. +concurrency: + group: sync-issues + cancel-in-progress: false + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - name: Sync issues/*.md to GitHub issues + run: scripts/sync-issues.sh + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 27644a1..d51c51e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,17 @@ The first release of the Tailwise inference engine. Langfuse, an OpenTelemetry Collector), each from its own durable queue. - Runs with no config file: the built-in defaults need only `JEV_API_KEY`. Manifests set the questions a model is asked and are optional. +- Five built-in checks asked alongside `answer_wrong` in the same call: + `unsupported_answer`, `missing_tool_calls`, `answered_on_failure`, + `ignored_instructions` and `contradicts_knowledge`. A trace is kept when any + of them reaches its threshold. +- A manifest `decision` block: `any_of` lists the questions that can keep a + trace, in priority order after the decision question, and `thresholds` + gives any of them its own threshold. +- Flags and severity on every decision: the questions that reached their + threshold, and a priority-weighted score that ranks kept traces for review. + Both appear in the decision log, with every answer, and on exported traces + as `eval.flags`, `eval.severity` and `eval.answers.`. - One model per agent, with its own questions, endpoint or key. - The engine's own metrics and errors over OpenTelemetry. - Docker Compose for a local stack, a Helm chart for Kubernetes, and diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0335058..2e0b1a4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -90,6 +90,26 @@ make test - User-facing behaviour changes are reflected in the README, and changes to configuration in `tailwise.schema.yaml` and the examples. +## Issues + +Planned work is written as Markdown in [issues/](issues/), one file per issue, +and reviewed like code. On every push to main that changes the folder, +[.github/workflows/issues.yml](.github/workflows/issues.yml) runs +[scripts/sync-issues.sh](scripts/sync-issues.sh), which keeps the GitHub +issues in step: + +- A new file creates an issue, and an edited file updates its issue's title + and body. Edit the file, not the issue: the next sync overwrites the issue. +- Deleting a file closes its issue with a comment. +- The first `# ` line is the title, an optional `**Labels:** a, b` line sets + labels (created if missing; labels are only ever added), and the rest is + the body. +- A hidden marker at the end of the body names the file, so renaming a file + creates a new issue and closes the old one. + +`DRY_RUN=1 scripts/sync-issues.sh` prints what a sync would do (it needs `gh` +signed in to the repository). + ## Commits, versioning and releases Tailwise follows [Semantic Versioning](https://semver.org), starting at diff --git a/README.md b/README.md index 1bde8f0..fc52e72 100644 --- a/README.md +++ b/README.md @@ -65,9 +65,35 @@ Requirements: - A Jev API key, in `JEV_API_KEY` - Docker, to run the engine, NATS and Phoenix with Compose -- Go 1.26 or newer, to build or test outside Docker +- Go 1.26 or newer, to build or test outside Docker (not for a release binary) - Python 3, only for `make samples` +### From a release binary + +Every release on [GitHub Releases](https://github.com/tecnosam/tailwise/releases) +has a `tailwise` binary for Linux, macOS and Windows, each on amd64 and +arm64, and a `checksums.txt`. The binary needs nothing else: the queues run +inside it unless `queue.url` names a NATS server. + +```sh +VERSION=0.1.0 OS=linux ARCH=amd64 # OS: linux, darwin; ARCH: amd64, arm64 +BASE=https://github.com/tecnosam/tailwise/releases/download/v$VERSION +curl -fsSLO "$BASE/tailwise_${VERSION}_${OS}_${ARCH}.tar.gz" +curl -fsSLO "$BASE/checksums.txt" +sha256sum --ignore-missing -c checksums.txt # on macOS: shasum -a 256 --ignore-missing -c checksums.txt +tar -xzf "tailwise_${VERSION}_${OS}_${ARCH}.tar.gz" tailwise + +export JEV_API_KEY=... +./tailwise -version +./tailwise # the built-in defaults: Jev, embedded NATS, Phoenix at localhost:6006 +``` + +Windows archives are `.zip` files holding `tailwise.exe`. On macOS, a binary +downloaded in a browser is quarantined; clear it with +`xattr -d com.apple.quarantine tailwise`. To see kept traces, start +Phoenix too (`make phoenix`, or `phoenix serve`); it takes port 4317, so run +the engine with `TAILWISE_OTLP_GRPC_ADDR=:14317`. + ### With Docker Compose ```sh diff --git a/scripts/sync-issues.sh b/scripts/sync-issues.sh new file mode 100755 index 0000000..acbdd79 --- /dev/null +++ b/scripts/sync-issues.sh @@ -0,0 +1,87 @@ +#!/bin/sh +# Syncs issues/*.md to GitHub issues: each file is one issue, created the first +# time and edited after. An issue whose file was deleted is closed. Run by +# .github/workflows/issues.yml on pushes to main. +# +# scripts/sync-issues.sh (needs gh, authenticated, and jq) +# DRY_RUN=1 scripts/sync-issues.sh prints what it would do +# +# A file's first "# " line is the issue title, an optional "**Labels:** a, b" +# line its labels, and the rest its body. The body ends with a hidden marker +# naming the file, which is how a file finds its issue again. +set -eu + +dir=${ISSUES_DIR:-issues} +dry=${DRY_RUN:-} +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +run() { + if [ -n "$dry" ]; then + echo "would run: $*" + else + "$@" + fi +} + +marker() { printf '' "$1"; } + +# Every issue, open or closed, with the file its marker names ("" for none). +# Fetched to a file first, so a failed fetch stops the script instead of +# looking like a repository with no issues. +gh issue list --state all --limit 1000 --json number,title,body,state > "$tmp/list.json" +jq '[.[] | . + {file: ((.body | capture("").f) // "")}]' "$tmp/list.json" > "$tmp/issues.json" + +for file in "$dir"/*.md; do + [ -e "$file" ] || continue + title=$(sed -n 's/^# //p' "$file" | head -n 1) + if [ -z "$title" ]; then + echo "skip $file: no '# ' title line" >&2 + continue + fi + labels=$(sed -n 's/^\*\*Labels:\*\* *//p' "$file" | head -n 1) + + # The body: everything but the title and labels lines, leading blank lines + # trimmed, then the marker. + awk -v t="# $title" ' + !titled && $0 == t { titled = 1; next } + /^\*\*Labels:\*\*/ && !labelled { labelled = 1; next } + { print } + ' "$file" | sed -e '/./,$!d' > "$tmp/body.md" + printf '\n%s\n' "$(marker "$file")" >> "$tmp/body.md" + + echo "$labels" | tr ',' '\n' | sed 's/^ *//; s/ *$//' | sed '/^$/d' > "$tmp/labels" + + number=$(jq -r --arg f "$file" '[.[] | select(.file == $f)][0].number // empty' "$tmp/issues.json") + if [ -z "$number" ]; then + echo "create: $title ($file)" + if [ -n "$dry" ]; then + number="(new)" + else + number=$(gh issue create --title "$title" --body-file "$tmp/body.md" | sed 's|.*/||') + fi + else + same=$(jq -r --arg f "$file" --arg t "$title" --rawfile b "$tmp/body.md" \ + '[.[] | select(.file == $f)][0] | (.title == $t and (.body | rtrimstr("\n")) == ($b | rtrimstr("\n")))' "$tmp/issues.json") + if [ "$same" = true ]; then + echo "unchanged: #$number $title" + else + echo "edit: #$number $title" + run gh issue edit "$number" --title "$title" --body-file "$tmp/body.md" + fi + fi + + # Labels are only added: one removed from the file stays on the issue. + while IFS= read -r l; do + run gh label create "$l" --force /dev/null + run gh issue edit "$number" --add-label "$l" /dev/null + done < "$tmp/labels" +done + +# Close open issues whose file is gone. +jq -r '.[] | select(.file != "" and .state == "OPEN") | "\(.number) \(.file)"' "$tmp/issues.json" | + while read -r number file; do + [ -e "$file" ] && continue + echo "close: #$number ($file was deleted)" + run gh issue close "$number" --comment "Closed because $file was removed from the repository."