Align harmony-go with official Harmony HEAD

.github/workflows/go.yml

@@ -21,6 +21,39 @@ env:
   GO_VERSION: '1.25.1'
 
 jobs:
+  parity:
+    name: Parity
+    runs-on: ubuntu-latest
+    env:
+      TIKTOKEN_GO_CACHE_DIR: ${{ github.workspace }}/.tiktoken-cache
+      GOCACHE: ${{ github.workspace }}/.gocache
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+      - name: Prepare directories
+        run: |
+          mkdir -p .tiktoken-cache .gocache
+      - name: Cache tokenizer data
+        uses: actions/cache@v4
+        with:
+          path: .tiktoken-cache
+          key: tiktoken-${{ hashFiles('go.mod', 'go.sum') }}
+      - name: Cache Go build cache
+        uses: actions/cache@v4
+        with:
+          path: .gocache
+          key: ${{ runner.os }}-go${{ env.GO_VERSION }}-build-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-go${{ env.GO_VERSION }}-build-
+      - name: Run mirrored parity suites
+        run: go test . ./benchmarks/go/golden
+
   lint:
     name: Lint
     runs-on: ubuntu-latest
@@ -144,7 +177,7 @@ jobs:
   arenas:
     name: Arenas Build
     runs-on: ubuntu-latest
-    needs: test
+    needs: [test, parity]
     # Skip this extra build on PRs; run on push/main or manual
     if: github.event_name != 'pull_request'
     env:
@@ -183,7 +216,7 @@ jobs:
     name: Benchmarks (manual)
     if: github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
-    needs: test
+    needs: [test, parity]
     env:
       TIKTOKEN_GO_CACHE_DIR: ${{ github.workspace }}/.tiktoken-cache
     steps:
@@ -230,3 +263,14 @@ jobs:
             profiles/bench-heap.txt
             profiles/bench-arenas.txt
           if-no-files-found: error
+
+  upstream-drift:
+    name: Upstream Drift
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+      - name: Check live upstream fixture drift
+        run: python3 scripts/check_upstream_fixture_drift.py

.gitignore

@@ -30,6 +30,7 @@ trace.out
 .gomodcache/
 .gopath/
 .gotmp/
+.worktrees/
 .tiktoken-cache/
 tmp/
 

CONTRIBUTING.md

@@ -24,6 +24,8 @@ Thanks for your interest in improving Harmony Go! This guide explains how to set
 ## Benchmarks (optional)
 To compare performance locally:
 - Go only: `go test -run '^$' -bench '^Benchmark' -benchmem ./benchmarks/go`
+- Parse baseline comparison: `python3 scripts/compare_parse_bench.py`
+- Live upstream fixture drift: `python3 scripts/check_upstream_fixture_drift.py`
 - See `docs/python_go_performance.md` for methodology and notes.
 
 ## Style & Conventions
@@ -34,6 +36,7 @@ To compare performance locally:
 ## Pull Requests
 - Describe the problem, the approach, and any trade‑offs.
 - Include before/after performance numbers when optimizing.
+- If parsing performance changes, include the output of `python3 scripts/compare_parse_bench.py`.
 - Link related issues and add screenshots for CLI UX changes if applicable.
 
 ### Requesting Full CI On PRs

README.md

@@ -47,8 +47,10 @@ It mirrors the reference semantics of the upstream implementation while offering
 
 ## Features
 - Render: `Render`, `RenderConversation`, `RenderConversationForCompletion`, `RenderConversationForTraining`.
-- Parse: `ParseMessagesFromCompletionTokens` for batch; `NewStreamParser` for incremental streaming.
-- Token helpers: `StopTokens`, `StopTokensForAssistantActions`, `DecodeUTF8`/`DecodeBytes`.
+- Parse: `ParseMessagesFromCompletionTokens`/`ParseMessagesFromCompletionTokensWithOptions` for batch; `NewStreamParser`/`NewStreamParserWithOptions` for incremental streaming.
+- Token helpers: `StopTokens`, `StopTokensForAssistantActions`, `DecodeUTF8`/`DecodeBytes`, `Decode`, `IsSpecialToken`, `AllSpecialTokens`.
+- Encode controls: `Encode` with `EncodeOptions` for upstream-compatible allowed/disallowed special token handling.
+- Built-in namespaces: `BrowserToolNamespace` and `PythonToolNamespace`.
 - Tools & channels: correct formatting tokens, `channel`, `recipient`, and `content_type` handling.
 - No external deps: ships with O200k tokenizer integration and Harmony specials.
 
@@ -110,7 +112,8 @@ echo '{"messages":[{"role":"user","content":[{"type":"text","text":"hello"}]}]}'
 echo '{"messages":[{"role":"assistant","channel":"final","content":[{"type":"text","text":"ok"}]}]}' | \
   harmony-go render-training
 
-# Parse tokens back to messages
+# Parse tokens back to messages (role hint is optional)
+echo '[1,2,3,4]' | harmony-go parse
 echo '[1,2,3,4]' | harmony-go parse -role assistant
 
 # Decode raw tokens into text (debugging)
@@ -121,6 +124,12 @@ echo '[200014]' | harmony-go decode
 
 Harmony Go is engineered for low allocations and high throughput. In local runs, it consistently outperforms the official Python bindings for render/parse‑heavy workloads. Streaming parse matches batch throughput while shaving a few allocations.
 
+Official implementations used for comparison
+
+- Python: the official `openai-harmony` Python bindings.
+- Rust: the official `openai/harmony` implementation and crate.
+- Upstream fixture/source-of-truth snapshot mirrored in this repo: `testdata/upstream-head/metadata.json`.
+
 Benchmarks below are from the cross‑parity harness in `benchmarks/python/bench.py`, Go benchmarks in `benchmarks/go`, and a Rust micro‑bench harness in `benchmarks/rust` at 200 iterations. Full details: `docs/python_go_performance.md`.
 
 Rendering
@@ -144,6 +153,27 @@ Parsing
 
 Observed speedups range from ~6.6× (large render) up to ≥190× (tool‑call parse).
 
+Current Go benchmark snapshot
+
+- Machine: Apple M2 Ultra, macOS, 200 iterations
+- Command: `TIKTOKEN_OFFLINE=1 go test -run '^$' -bench '^Benchmark' -benchmem -benchtime=200x ./benchmarks/go`
+- Date: March 20, 2026
+
+| Benchmark | ns/op | B/op | allocs/op |
+| --- | ---: | ---: | ---: |
+| Render tool-call | 3,552 | 942 | 11 |
+| Render large (auto-drop) | 310,770 | 466,192 | 4,850 |
+| Render large (keep analysis) | 475,765 | 939,856 | 9,681 |
+| Parse tool-call | 2,056 | 2,040 | 52 |
+| Stream parse tool-call | 1,842 | 2,040 | 52 |
+| Parse large completion | 17,073 | 26,184 | 465 |
+| Stream parse large completion | 20,120 | 26,184 | 465 |
+
+Parse benchmark drift guard
+
+- Compare the current parse subset against the checked-in baseline with `python3 scripts/compare_parse_bench.py`.
+- Verify mirrored upstream Harmony fixtures still match live upstream `HEAD` with `python3 scripts/check_upstream_fixture_drift.py`.
+
 Rust vs Go (micro‑benches)
 
 - Machine: Apple M2 Ultra, macOS, 200 iterations (Sep 21, 2025)
@@ -187,6 +217,8 @@ Build with `GOEXPERIMENT=arenas` to place decoder storage in an arena. This prim
 ## Testing & Parity
 - Unit tests: `go test ./...`
 - Benchmarks: `go test -run '^$' -bench '^Benchmark' -benchmem ./benchmarks/go`
+- Parse benchmark comparison: `python3 scripts/compare_parse_bench.py`
+- Live upstream fixture drift: `python3 scripts/check_upstream_fixture_drift.py`
 - Lint: `golangci-lint run ./...`
 - Coverage: `go test ./... -cover -coverprofile=coverage.out && go tool cover -func=coverage.out`
 - Python parity tests (optional): ensure `openai_harmony` is installed, then run pytest in `tests/` to validate CLI parity.
@@ -196,6 +228,7 @@ What we check in parity tests:
 - Conversation rendering (system/developer/tools/channels) vs Python.
 - Training substitution (<|end|> → <|return|> for assistant:final).
 - Parser and stream parser produce the same message shapes as Python.
+- Mirrored official upstream fixtures stay pinned and checked for live `HEAD` drift.
 
 ## Configuration
 - `TIKTOKEN_ENCODINGS_BASE` — directory containing `o200k_base.tiktoken` (overrides remote).

benchmarks/go/golden/testdata/autodrop_false.tokens.json

@@ -3,10 +3,8 @@
   1428,
   200008,
   176289,
-  220,
-  3086,
-  220,
-  27176,
+  290,
+  10331,
   200007,
   200006,
   173781,
@@ -15,15 +13,12 @@
   200008,
   30377,
   289,
-  220,
-  258,
-  220,
-  31705,
+  306,
+  7408,
   200007,
   200006,
   173781,
-  220,
-  935,
+  316,
   28,
   44580,
   16128,
@@ -32,8 +27,7 @@
   815,
   200008,
   17952,
-  220,
-  9925,
+  2421,
   200012,
   200006,
   44580,
@@ -47,9 +41,7 @@
   17196,
   200008,
   12253,
-  220,
-  13320,
-  220,
-  2319,
+  481,
+  810,
   200007
 ]

benchmarks/go/golden/testdata/autodrop_true.tokens.json

@@ -3,15 +3,12 @@
   1428,
   200008,
   176289,
-  220,
-  3086,
-  220,
-  27176,
+  290,
+  10331,
   200007,
   200006,
   173781,
-  220,
-  935,
+  316,
   28,
   44580,
   16128,
@@ -20,8 +17,7 @@
   815,
   200008,
   17952,
-  220,
-  9925,
+  2421,
   200012,
   200006,
   44580,
@@ -35,9 +31,7 @@
   17196,
   200008,
   12253,
-  220,
-  13320,
-  220,
-  2319,
+  481,
+  810,
   200007
 ]