diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 0113ab8..c7a3d81 100755 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -21,6 +21,39 @@ env: GO_VERSION: '1.25.1' jobs: + parity: + name: Parity + runs-on: ubuntu-latest + env: + TIKTOKEN_GO_CACHE_DIR: ${{ github.workspace }}/.tiktoken-cache + GOCACHE: ${{ github.workspace }}/.gocache + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + - name: Prepare directories + run: | + mkdir -p .tiktoken-cache .gocache + - name: Cache tokenizer data + uses: actions/cache@v4 + with: + path: .tiktoken-cache + key: tiktoken-${{ hashFiles('go.mod', 'go.sum') }} + - name: Cache Go build cache + uses: actions/cache@v4 + with: + path: .gocache + key: ${{ runner.os }}-go${{ env.GO_VERSION }}-build-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-go${{ env.GO_VERSION }}-build- + - name: Run mirrored parity suites + run: go test . ./benchmarks/go/golden + lint: name: Lint runs-on: ubuntu-latest @@ -144,7 +177,7 @@ jobs: arenas: name: Arenas Build runs-on: ubuntu-latest - needs: test + needs: [test, parity] # Skip this extra build on PRs; run on push/main or manual if: github.event_name != 'pull_request' env: @@ -183,7 +216,7 @@ jobs: name: Benchmarks (manual) if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest - needs: test + needs: [test, parity] env: TIKTOKEN_GO_CACHE_DIR: ${{ github.workspace }}/.tiktoken-cache steps: @@ -230,3 +263,14 @@ jobs: profiles/bench-heap.txt profiles/bench-arenas.txt if-no-files-found: error + + upstream-drift: + name: Upstream Drift + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: Check live upstream fixture drift + run: python3 scripts/check_upstream_fixture_drift.py diff --git a/.gitignore b/.gitignore index 4f9a0a4..5d474c5 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ trace.out .gomodcache/ .gopath/ .gotmp/ +.worktrees/ .tiktoken-cache/ tmp/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e58fe89..623d5f6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,6 +24,8 @@ Thanks for your interest in improving Harmony Go! This guide explains how to set ## Benchmarks (optional) To compare performance locally: - Go only: `go test -run '^$' -bench '^Benchmark' -benchmem ./benchmarks/go` +- Parse baseline comparison: `python3 scripts/compare_parse_bench.py` +- Live upstream fixture drift: `python3 scripts/check_upstream_fixture_drift.py` - See `docs/python_go_performance.md` for methodology and notes. ## Style & Conventions @@ -34,6 +36,7 @@ To compare performance locally: ## Pull Requests - Describe the problem, the approach, and any trade‑offs. - Include before/after performance numbers when optimizing. +- If parsing performance changes, include the output of `python3 scripts/compare_parse_bench.py`. - Link related issues and add screenshots for CLI UX changes if applicable. ### Requesting Full CI On PRs diff --git a/README.md b/README.md index 06f3e6c..dd67bdf 100755 --- a/README.md +++ b/README.md @@ -47,8 +47,10 @@ It mirrors the reference semantics of the upstream implementation while offering ## Features - Render: `Render`, `RenderConversation`, `RenderConversationForCompletion`, `RenderConversationForTraining`. -- Parse: `ParseMessagesFromCompletionTokens` for batch; `NewStreamParser` for incremental streaming. -- Token helpers: `StopTokens`, `StopTokensForAssistantActions`, `DecodeUTF8`/`DecodeBytes`. +- Parse: `ParseMessagesFromCompletionTokens`/`ParseMessagesFromCompletionTokensWithOptions` for batch; `NewStreamParser`/`NewStreamParserWithOptions` for incremental streaming. +- Token helpers: `StopTokens`, `StopTokensForAssistantActions`, `DecodeUTF8`/`DecodeBytes`, `Decode`, `IsSpecialToken`, `AllSpecialTokens`. +- Encode controls: `Encode` with `EncodeOptions` for upstream-compatible allowed/disallowed special token handling. +- Built-in namespaces: `BrowserToolNamespace` and `PythonToolNamespace`. - Tools & channels: correct formatting tokens, `channel`, `recipient`, and `content_type` handling. - No external deps: ships with O200k tokenizer integration and Harmony specials. @@ -110,7 +112,8 @@ echo '{"messages":[{"role":"user","content":[{"type":"text","text":"hello"}]}]}' echo '{"messages":[{"role":"assistant","channel":"final","content":[{"type":"text","text":"ok"}]}]}' | \ harmony-go render-training -# Parse tokens back to messages +# Parse tokens back to messages (role hint is optional) +echo '[1,2,3,4]' | harmony-go parse echo '[1,2,3,4]' | harmony-go parse -role assistant # Decode raw tokens into text (debugging) @@ -121,6 +124,12 @@ echo '[200014]' | harmony-go decode Harmony Go is engineered for low allocations and high throughput. In local runs, it consistently outperforms the official Python bindings for render/parse‑heavy workloads. Streaming parse matches batch throughput while shaving a few allocations. +Official implementations used for comparison + +- Python: the official `openai-harmony` Python bindings. +- Rust: the official `openai/harmony` implementation and crate. +- Upstream fixture/source-of-truth snapshot mirrored in this repo: `testdata/upstream-head/metadata.json`. + Benchmarks below are from the cross‑parity harness in `benchmarks/python/bench.py`, Go benchmarks in `benchmarks/go`, and a Rust micro‑bench harness in `benchmarks/rust` at 200 iterations. Full details: `docs/python_go_performance.md`. Rendering @@ -144,6 +153,27 @@ Parsing Observed speedups range from ~6.6× (large render) up to ≥190× (tool‑call parse). +Current Go benchmark snapshot + +- Machine: Apple M2 Ultra, macOS, 200 iterations +- Command: `TIKTOKEN_OFFLINE=1 go test -run '^$' -bench '^Benchmark' -benchmem -benchtime=200x ./benchmarks/go` +- Date: March 20, 2026 + +| Benchmark | ns/op | B/op | allocs/op | +| --- | ---: | ---: | ---: | +| Render tool-call | 3,552 | 942 | 11 | +| Render large (auto-drop) | 310,770 | 466,192 | 4,850 | +| Render large (keep analysis) | 475,765 | 939,856 | 9,681 | +| Parse tool-call | 2,056 | 2,040 | 52 | +| Stream parse tool-call | 1,842 | 2,040 | 52 | +| Parse large completion | 17,073 | 26,184 | 465 | +| Stream parse large completion | 20,120 | 26,184 | 465 | + +Parse benchmark drift guard + +- Compare the current parse subset against the checked-in baseline with `python3 scripts/compare_parse_bench.py`. +- Verify mirrored upstream Harmony fixtures still match live upstream `HEAD` with `python3 scripts/check_upstream_fixture_drift.py`. + Rust vs Go (micro‑benches) - Machine: Apple M2 Ultra, macOS, 200 iterations (Sep 21, 2025) @@ -187,6 +217,8 @@ Build with `GOEXPERIMENT=arenas` to place decoder storage in an arena. This prim ## Testing & Parity - Unit tests: `go test ./...` - Benchmarks: `go test -run '^$' -bench '^Benchmark' -benchmem ./benchmarks/go` +- Parse benchmark comparison: `python3 scripts/compare_parse_bench.py` +- Live upstream fixture drift: `python3 scripts/check_upstream_fixture_drift.py` - Lint: `golangci-lint run ./...` - Coverage: `go test ./... -cover -coverprofile=coverage.out && go tool cover -func=coverage.out` - Python parity tests (optional): ensure `openai_harmony` is installed, then run pytest in `tests/` to validate CLI parity. @@ -196,6 +228,7 @@ What we check in parity tests: - Conversation rendering (system/developer/tools/channels) vs Python. - Training substitution (<|end|> → <|return|> for assistant:final). - Parser and stream parser produce the same message shapes as Python. +- Mirrored official upstream fixtures stay pinned and checked for live `HEAD` drift. ## Configuration - `TIKTOKEN_ENCODINGS_BASE` — directory containing `o200k_base.tiktoken` (overrides remote). diff --git a/benchmarks/go/golden/testdata/autodrop_false.tokens.json b/benchmarks/go/golden/testdata/autodrop_false.tokens.json index df5c649..b72c374 100644 --- a/benchmarks/go/golden/testdata/autodrop_false.tokens.json +++ b/benchmarks/go/golden/testdata/autodrop_false.tokens.json @@ -3,10 +3,8 @@ 1428, 200008, 176289, - 220, - 3086, - 220, - 27176, + 290, + 10331, 200007, 200006, 173781, @@ -15,15 +13,12 @@ 200008, 30377, 289, - 220, - 258, - 220, - 31705, + 306, + 7408, 200007, 200006, 173781, - 220, - 935, + 316, 28, 44580, 16128, @@ -32,8 +27,7 @@ 815, 200008, 17952, - 220, - 9925, + 2421, 200012, 200006, 44580, @@ -47,9 +41,7 @@ 17196, 200008, 12253, - 220, - 13320, - 220, - 2319, + 481, + 810, 200007 ] diff --git a/benchmarks/go/golden/testdata/autodrop_true.tokens.json b/benchmarks/go/golden/testdata/autodrop_true.tokens.json index d04886a..d951cac 100644 --- a/benchmarks/go/golden/testdata/autodrop_true.tokens.json +++ b/benchmarks/go/golden/testdata/autodrop_true.tokens.json @@ -3,15 +3,12 @@ 1428, 200008, 176289, - 220, - 3086, - 220, - 27176, + 290, + 10331, 200007, 200006, 173781, - 220, - 935, + 316, 28, 44580, 16128, @@ -20,8 +17,7 @@ 815, 200008, 17952, - 220, - 9925, + 2421, 200012, 200006, 44580, @@ -35,9 +31,7 @@ 17196, 200008, 12253, - 220, - 13320, - 220, - 2319, + 481, + 810, 200007 ] diff --git a/benchmarks/go/golden/testdata/system_developer.tokens.json b/benchmarks/go/golden/testdata/system_developer.tokens.json index 3cb1dba..90cbce8 100755 --- a/benchmarks/go/golden/testdata/system_developer.tokens.json +++ b/benchmarks/go/golden/testdata/system_developer.tokens.json @@ -7,9 +7,7 @@ 31542, 198, 87447, - 220, - 9804, - 2864, + 100594, 25, 220, 1323, @@ -20,8 +18,7 @@ 2290, 198, 6576, - 220, - 1319, + 3521, 25, 220, 1323, @@ -34,40 +31,29 @@ 30377, 289, 25, - 220, - 54482, + 14093, 279, 2, - 220, - 25128, + 20574, 279, 877, - 220, - 44580, + 9964, 279, 393, - 220, - 4701, - 220, - 8259, - 220, - 44580, + 9238, + 8205, + 9964, 198, 4797, - 220, - 44580, + 9964, 405, 3914, - 220, - 34681, - 220, - 3086, - 220, - 120983, + 87964, + 290, + 22888, 198, 2493, - 220, - 522, + 717, 170154, 314, 11350, @@ -75,73 +61,46 @@ 405, 7693, 8528, - 220, - 1655, + 1621, 412, 9263, 871, - 220, - 1252, + 1062, 307, 198, 92, 602, - 220, - 4797, - 220, - 44580, + 9819, + 9964, 279, 2, - 220, - 5258, - 220, - 76366, + 13888, + 18403, 25, - 220, - 35644, + 8450, 11, - 220, - 12606, - 815, + 49159, 11, - 220, - 17196, + 1721, 13, - 220, - 10135, - 220, - 46036, - 220, - 1464, - 220, - 159353, - 220, - 1938, - 220, - 50492, - 220, - 3938, + 21030, + 2804, + 413, + 7360, + 395, + 1753, + 3176, 558, 63446, - 220, - 935, - 220, - 78079, - 220, - 30616, - 220, - 46036, - 220, - 2319, - 220, - 935, - 220, - 3086, - 220, - 12606, - 815, - 220, - 21453, + 316, + 1879, + 8437, + 2804, + 810, + 316, + 290, + 49159, + 9334, 25, 461, 44580, @@ -151,39 +110,29 @@ 77944, 200008, 2, - 220, - 61111, + 68406, 279, 125928, - 220, - 28275, - 220, - 54065, + 35160, + 13306, 558, 198, 2, - 220, - 25128, + 20574, 279, 877, - 220, - 44580, + 9964, 279, 4797, - 220, - 44580, + 9964, 405, 3914, - 220, - 34681, - 220, - 3086, - 220, - 120983, + 87964, + 290, + 22888, 198, 2493, - 220, - 522, + 717, 170154, 314, 11350, @@ -191,46 +140,31 @@ 405, 7693, 8528, - 220, - 1655, + 1621, 412, 9263, 871, - 220, - 1252, + 1062, 307, 198, 92, 602, - 220, - 4797, - 220, - 44580, + 9819, + 9964, 200007, 200006, 1428, 200008, 15274, - 220, - 64, - 220, - 1635, - 220, - 258, - 220, - 28499, - 220, - 106672, - 13009, - 220, - 6332, - 220, - 18646, - 220, - 427, - 220, - 82, - 3635, + 261, + 2163, + 306, + 6610, + 18826, + 483, + 4232, + 326, + 58545, 13, 200007, 200006, diff --git a/benchmarks/go/golden/testdata/tool_call.tokens.json b/benchmarks/go/golden/testdata/tool_call.tokens.json index 3efdcc5..d5c1ca2 100755 --- a/benchmarks/go/golden/testdata/tool_call.tokens.json +++ b/benchmarks/go/golden/testdata/tool_call.tokens.json @@ -3,16 +3,11 @@ 1428, 200008, 4827, - 220, - 276, - 220, - 3086, - 220, - 28393, - 220, - 258, - 220, - 46661, + 382, + 290, + 11122, + 306, + 38371, 30, 200007, 200006, @@ -21,41 +16,28 @@ 35644, 200008, 1844, - 220, - 9523, + 31064, 25, 392, 4827, - 220, - 276, - 220, - 3086, - 220, - 28393, - 220, - 258, - 220, - 46661, + 382, + 290, + 11122, + 306, + 38371, 16842, - 220, - 2167, - 220, - 24630, - 220, - 935, - 220, - 1821, - 220, - 29712, + 1416, + 1309, + 316, + 1199, + 37342, 170154, - 220, - 17952, + 4584, 13, 200007, 200006, 173781, - 220, - 935, + 316, 28, 44580, 76043, @@ -72,9 +54,7 @@ 1243, 392, 28499, - 220, - 106672, - 13009, + 18826, 18583, 200012, 200006, diff --git a/builtins.go b/builtins.go new file mode 100644 index 0000000..627e9b2 --- /dev/null +++ b/builtins.go @@ -0,0 +1,45 @@ +package harmony + +import "encoding/json" + +// BrowserToolNamespace returns the built-in browser namespace used by official Harmony prompts. +func BrowserToolNamespace() ToolNamespaceConfig { + exampleOne := "`\u30106\u2020L9-L11\u3011`" + exampleTwo := "`\u30108\u2020L3\u3011`" + desc := "Tool for browsing.\n" + + "The `cursor` appears in brackets before each browsing display: `[{cursor}]`.\n" + + "Cite information from the tool using the following format:\n" + + "`【{cursor}†L{line_start}(-L{line_end})?】`, for example: " + exampleOne + " or " + exampleTwo + ".\n" + + "Do not quote more than 10 words directly from the tool output.\n" + + "sources=web (default: web)" + return ToolNamespaceConfig{ + Name: "browser", + Description: &desc, + Tools: []ToolDescription{ + { + Name: "search", + Description: "Searches for information related to `query` and displays `topn` results.", + Parameters: json.RawMessage(`{"type":"object","properties":{"query":{"type":"string"},"topn":{"type":"number","default":10},"source":{"type":"string"}},"required":["query"]}`), + }, + { + Name: "open", + Description: "Opens the link `id` from the page indicated by `cursor` starting at line number `loc`, showing `num_lines` lines.\nValid link ids are displayed with the formatting: `【{id}†.*】`.\nIf `cursor` is not provided, the most recent page is implied.\nIf `id` is a string, it is treated as a fully qualified URL associated with `source`.\nIf `loc` is not provided, the viewport will be positioned at the beginning of the document or centered on the most relevant passage, if available.\nUse this function without `id` to scroll to a new location of an opened page.", + Parameters: json.RawMessage(`{"type":"object","properties":{"id":{"type":["number","string"],"default":-1},"cursor":{"type":"number","default":-1},"loc":{"type":"number","default":-1},"num_lines":{"type":"number","default":-1},"view_source":{"type":"boolean","default":false},"source":{"type":"string"}}}`), + }, + { + Name: "find", + Description: "Finds exact matches of `pattern` in the current page, or the page given by `cursor`.", + Parameters: json.RawMessage(`{"type":"object","properties":{"pattern":{"type":"string"},"cursor":{"type":"number","default":-1}},"required":["pattern"]}`), + }, + }, + } +} + +// PythonToolNamespace returns the built-in python namespace used by official Harmony prompts. +func PythonToolNamespace() ToolNamespaceConfig { + desc := "Use this tool to execute Python code in your chain of thought. The code will not be shown to the user. This tool should be used for internal reasoning, but not for code that is intended to be visible to the user (e.g. when creating plots, tables, or files).\n\nWhen you send a message containing Python code to python, it will be executed in a stateful Jupyter notebook environment. python will respond with the output of the execution or time out after 120.0 seconds. The drive at '/mnt/data' can be used to save and persist user files. Internet access for this session is UNKNOWN. Depends on the cluster." + return ToolNamespaceConfig{ + Name: "python", + Description: &desc, + } +} diff --git a/capability_parity_test.go b/capability_parity_test.go new file mode 100644 index 0000000..a1f3175 --- /dev/null +++ b/capability_parity_test.go @@ -0,0 +1,387 @@ +package harmony + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/euforicio/harmony-go/tokenizer" +) + +func readUpstreamFixture(t *testing.T, name string) string { + t.Helper() + path := filepath.Join("testdata", "upstream-head", name) + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile(%s): %v", path, err) + } + return strings.TrimRight(string(data), "\n") +} + +func mustJSON(t *testing.T, v any) string { + t.Helper() + data, err := json.Marshal(v) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + return string(data) +} + +func TestRenderNoToolsFixture(t *testing.T) { + enc := mustEncoding(t) + + conv := Conversation{Messages: []Message{ + { + Author: Author{Role: RoleSystem}, + Content: []Content{{ + Type: ContentSystem, + System: &SystemContent{ + ConversationStartDate: strPtr("2025-06-28"), + }, + }}, + }, + }} + + tokens, err := enc.RenderConversationForCompletion(conv, RoleAssistant, nil) + if err != nil { + t.Fatalf("RenderConversationForCompletion: %v", err) + } + got, err := enc.DecodeUTF8(tokens) + if err != nil { + t.Fatalf("DecodeUTF8: %v", err) + } + want := readUpstreamFixture(t, "test_no_tools.txt") + if got != want { + t.Fatalf("render mismatch\n got:\n%s\nwant:\n%s", got, want) + } +} + +func TestRenderBrowserToolOnlyFixture(t *testing.T) { + enc := mustEncoding(t) + + tools := map[string]ToolNamespaceConfig{ + "browser": BrowserToolNamespace(), + } + conv := Conversation{Messages: []Message{ + { + Author: Author{Role: RoleSystem}, + Content: []Content{{ + Type: ContentSystem, + System: &SystemContent{ + ConversationStartDate: strPtr("2025-06-28"), + Tools: tools, + }, + }}, + }, + }} + + tokens, err := enc.RenderConversationForCompletion(conv, RoleAssistant, nil) + if err != nil { + t.Fatalf("RenderConversationForCompletion: %v", err) + } + got, err := enc.DecodeUTF8(tokens) + if err != nil { + t.Fatalf("DecodeUTF8: %v", err) + } + want := readUpstreamFixture(t, "test_browser_tool_only.txt") + if got != want { + t.Fatalf("render mismatch\n got:\n%s\nwant:\n%s", got, want) + } +} + +func TestRenderBrowserAndPythonToolFixture(t *testing.T) { + enc := mustEncoding(t) + + tools := map[string]ToolNamespaceConfig{ + "browser": BrowserToolNamespace(), + "python": PythonToolNamespace(), + } + conv := Conversation{Messages: []Message{ + { + Author: Author{Role: RoleSystem}, + Content: []Content{{ + Type: ContentSystem, + System: &SystemContent{ + ConversationStartDate: strPtr("2025-06-28"), + Tools: tools, + }, + }}, + }, + }} + + tokens, err := enc.RenderConversationForCompletion(conv, RoleAssistant, nil) + if err != nil { + t.Fatalf("RenderConversationForCompletion: %v", err) + } + got, err := enc.DecodeUTF8(tokens) + if err != nil { + t.Fatalf("DecodeUTF8: %v", err) + } + want := readUpstreamFixture(t, "test_browser_and_python_tool.txt") + if got != want { + t.Fatalf("render mismatch\n got:\n%s\nwant:\n%s", got, want) + } +} + +func TestKeepAnalysisBetweenFinalMessages(t *testing.T) { + enc := mustEncoding(t) + + conv := Conversation{Messages: []Message{ + {Author: Author{Role: RoleUser}, Content: []Content{{Type: ContentText, Text: "What is 2 + 2?"}}}, + {Author: Author{Role: RoleAssistant}, Channel: "analysis", Content: []Content{{Type: ContentText, Text: "thinking 2+2"}}}, + {Author: Author{Role: RoleAssistant}, Channel: "final", Content: []Content{{Type: ContentText, Text: "4"}}}, + {Author: Author{Role: RoleUser}, Content: []Content{{Type: ContentText, Text: "What is 3 + 5?"}}}, + {Author: Author{Role: RoleAssistant}, Channel: "analysis", Content: []Content{{Type: ContentText, Text: "thinking 3+5"}}}, + {Author: Author{Role: RoleAssistant}, Channel: "final", Content: []Content{{Type: ContentText, Text: "8"}}}, + }} + + tokens, err := enc.RenderConversation(conv, nil) + if err != nil { + t.Fatalf("RenderConversation: %v", err) + } + got, err := enc.DecodeUTF8(tokens) + if err != nil { + t.Fatalf("DecodeUTF8: %v", err) + } + want := readUpstreamFixture(t, "test_keep_analysis_between_finals.txt") + if got != want { + t.Fatalf("render mismatch\n got:\n%s\nwant:\n%s", got, want) + } +} + +func TestEncodeAllowedDisallowedSpecial(t *testing.T) { + enc := mustEncoding(t) + + hello, err := enc.Encode("hello world", nil) + if err != nil { + t.Fatalf("Encode ordinary: %v", err) + } + if want := []uint32{24912, 2375}; !slices.Equal(hello, want) { + t.Fatalf("ordinary encode mismatch\n got: %v\nwant: %v", hello, want) + } + + allowed := map[string]struct{}{"<|start|>": {}} + got, err := enc.Encode("<|start|>", &EncodeOptions{AllowedSpecial: allowed}) + if err != nil { + t.Fatalf("Encode allowed special: %v", err) + } + if want := []uint32{tokenizer.TokStart}; !slices.Equal(got, want) { + t.Fatalf("allowed special mismatch\n got: %v\nwant: %v", got, want) + } + + if _, err := enc.Encode("<|start|>", nil); err == nil { + t.Fatalf("expected disallowed special error") + } + + got, err = enc.Encode("<|start|>", &EncodeOptions{DisallowedSpecial: map[string]struct{}{}}) + if err != nil { + t.Fatalf("Encode natural special text: %v", err) + } + wantNatural := []uint32{27, 91, 5236, 91, 29} + if !slices.Equal(got, wantNatural) { + t.Fatalf("natural special mismatch\n got: %v\nwant: %v", got, wantNatural) + } +} + +func TestDecodeReplaceAndIsSpecialToken(t *testing.T) { + enc := mustEncoding(t) + + if !enc.IsSpecialToken(tokenizer.TokStart) { + t.Fatalf("TokStart should be special") + } + if enc.IsSpecialToken(24912) { + t.Fatalf("24912 should not be special") + } + + if got := enc.Decode([]uint32{24912, 2375}); got != "hello world" { + t.Fatalf("Decode ordinary = %q", got) + } + + if _, err := enc.DecodeUTF8([]uint32{132990, 9552}); err == nil { + t.Fatalf("expected invalid utf-8 error") + } + if got := enc.Decode([]uint32{132990, 9552}); !strings.Contains(got, "Chicken") { + t.Fatalf("Decode should replace invalid utf-8, got %q", got) + } +} + +func TestParseMessagesWithOptionsNonStrictRecovers(t *testing.T) { + enc := mustEncoding(t) + + tokens, err := enc.Encode("I must refuse<|end|><|start|>assistant<|channel|>analysis<|message|>We must refuse<|end|>", &EncodeOptions{ + AllowedSpecial: AllSpecialTokens(), + }) + if err != nil { + t.Fatalf("Encode: %v", err) + } + + role := RoleAssistant + if _, err := enc.ParseMessagesFromCompletionTokensWithOptions(tokens, &role, ParseOptions{Strict: true}); err == nil { + t.Fatalf("expected strict parse failure") + } + + msgs, err := enc.ParseMessagesFromCompletionTokensWithOptions(tokens, &role, ParseOptions{Strict: false}) + if err != nil { + t.Fatalf("ParseMessagesFromCompletionTokensWithOptions non-strict: %v", err) + } + if len(msgs) != 2 { + t.Fatalf("expected 2 messages, got %d", len(msgs)) + } + if msgs[0].Content[0].Text != "I must refuse" { + t.Fatalf("unexpected recovered content: %+v", msgs[0]) + } +} + +func TestStreamParserWithOptionsInvalidUTF8Recovery(t *testing.T) { + enc := mustEncoding(t) + + prefix, err := enc.Encode("<|start|>assistant<|message|>", &EncodeOptions{AllowedSpecial: AllSpecialTokens()}) + if err != nil { + t.Fatalf("Encode prefix: %v", err) + } + suffix, err := enc.Encode("worked<|end|>", &EncodeOptions{AllowedSpecial: AllSpecialTokens()}) + if err != nil { + t.Fatalf("Encode suffix: %v", err) + } + tokens := append(append(prefix, 9552, 9552), suffix...) + + parser, err := NewStreamParserWithOptions(enc, nil, ParseOptions{Strict: true}) + if err != nil { + t.Fatalf("NewStreamParserWithOptions: %v", err) + } + for _, tok := range tokens { + if err := parser.Process(tok); err != nil { + t.Fatalf("Process: %v", err) + } + } + msgs := parser.Messages() + if len(msgs) != 1 { + t.Fatalf("expected 1 message, got %d", len(msgs)) + } + if msgs[0].Content[0].Text != " \uFFFD \uFFFDworked" { + t.Fatalf("unexpected recovered content: %q", msgs[0].Content[0].Text) + } +} + +func TestStreamParserNonStrictRoleHintDoesNotDuplicateTokens(t *testing.T) { + enc := mustEncoding(t) + + tokens, err := enc.Encode("second<|end|>", &EncodeOptions{ + AllowedSpecial: AllSpecialTokens(), + }) + if err != nil { + t.Fatalf("Encode: %v", err) + } + + role := RoleAssistant + parser, err := NewStreamParserWithOptions(enc, &role, ParseOptions{Strict: false}) + if err != nil { + t.Fatalf("NewStreamParserWithOptions: %v", err) + } + parser.state = stExpectStart + for _, tok := range tokens { + if err := parser.Process(tok); err != nil { + t.Fatalf("Process: %v", err) + } + } + if err := parser.ProcessEOS(); err != nil { + t.Fatalf("ProcessEOS: %v", err) + } + + if got := parser.Tokens(); !slices.Equal(got, tokens) { + t.Fatalf("Tokens() duplicated or dropped tokens: got %v want %v", got, tokens) + } + msgs := parser.Messages() + if len(msgs) != 1 { + t.Fatalf("expected 1 message, got %d", len(msgs)) + } + if msgs[0].Content[0].Text != "second" { + t.Fatalf("unexpected recovered messages: %+v", msgs) + } +} + +func TestStreamParserStrictRejectsStopTokenInHeader(t *testing.T) { + enc := mustEncoding(t) + + tokens, err := enc.Encode("<|start|>assistant<|end|>", &EncodeOptions{ + AllowedSpecial: AllSpecialTokens(), + }) + if err != nil { + t.Fatalf("Encode: %v", err) + } + + parser, err := NewStreamParserWithOptions(enc, nil, ParseOptions{Strict: true}) + if err != nil { + t.Fatalf("NewStreamParserWithOptions: %v", err) + } + for _, tok := range tokens { + err = parser.Process(tok) + if err != nil { + break + } + } + if err == nil || !strings.Contains(err.Error(), "unexpected stop token in message header") { + t.Fatalf("expected strict header stop error, got %v", err) + } +} + +func TestStreamParserNonStrictRecoveryClearsRoleHint(t *testing.T) { + enc := mustEncoding(t) + + tokens, err := enc.Encode("broken<|end|><|start|>functions.lookup_weather<|message|>{\"ok\":true}<|end|>", &EncodeOptions{ + AllowedSpecial: AllSpecialTokens(), + }) + if err != nil { + t.Fatalf("Encode: %v", err) + } + + role := RoleAssistant + parser, err := NewStreamParserWithOptions(enc, &role, ParseOptions{Strict: false}) + if err != nil { + t.Fatalf("NewStreamParserWithOptions: %v", err) + } + for _, tok := range tokens { + if err := parser.Process(tok); err != nil { + t.Fatalf("Process: %v", err) + } + } + if err := parser.ProcessEOS(); err != nil { + t.Fatalf("ProcessEOS: %v", err) + } + + msgs := parser.Messages() + if len(msgs) != 2 { + t.Fatalf("expected 2 messages, got %d", len(msgs)) + } + if msgs[0].Author.Role != RoleAssistant { + t.Fatalf("unexpected recovered role: %+v", msgs[0].Author) + } + if msgs[1].Author.Role != RoleTool || msgs[1].Author.Name != "functions.lookup_weather" { + t.Fatalf("expected tool message after recovery, got %+v", msgs[1].Author) + } +} + +func TestCLIParseRoleIsOptional(t *testing.T) { + enc := mustEncoding(t) + + text := "<|start|>browser.search<|message|>{\"ok\":true}<|end|>" + tokens, err := enc.Encode(text, &EncodeOptions{AllowedSpecial: AllSpecialTokens()}) + if err != nil { + t.Fatalf("Encode: %v", err) + } + + cmd := exec.Command("go", "run", "./cmd/harmony-go", "parse") + cmd.Dir = "." + cmd.Stdin = strings.NewReader(mustJSON(t, tokens)) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("go run parse: %v\n%s", err, out) + } + if !strings.Contains(string(out), "\"name\":\"browser.search\"") { + t.Fatalf("expected tool name in parse output, got %s", out) + } +} diff --git a/cmd/harmony-go/main.go b/cmd/harmony-go/main.go index 81e07aa..f4cd554 100755 --- a/cmd/harmony-go/main.go +++ b/cmd/harmony-go/main.go @@ -94,7 +94,7 @@ func main() { _ = json.NewEncoder(os.Stdout).Encode(tok) case "parse": fs := flag.NewFlagSet("parse", flag.ExitOnError) - role := fs.String("role", "assistant", "optional starting role (user|assistant|system|developer|tool)") + role := fs.String("role", "", "optional starting role (user|assistant|system|developer|tool)") _ = fs.Parse(os.Args[2:]) enc, err := harmony.LoadEncoding(harmony.HarmonyGptOss) if err != nil { diff --git a/docs/CI.md b/docs/CI.md index 5fefaba..733b5da 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -29,6 +29,8 @@ Pushes to `main`: Benchmarks: - Benchmarks are manual via the workflow_dispatch event. They build test binaries and run a subset of representative benchmarks; results are uploaded as artifacts. +- For local parse regression checks, run `python3 scripts/compare_parse_bench.py` against `benchmarks/python/results/go_bench.txt`. +- For live upstream parity drift, run `python3 scripts/check_upstream_fixture_drift.py`. Notes: diff --git a/docs/python_go_performance.md b/docs/python_go_performance.md index 8cc5ca6..51c31ae 100755 --- a/docs/python_go_performance.md +++ b/docs/python_go_performance.md @@ -23,6 +23,7 @@ This document compares the behaviour of the Python bindings (`openai_harmony`) w - **Large completion parse**: decoding a long assistant completion containing analysis and final channels. - **Streamable parser variants**: replaying tokens through `StreamableParser` to measure incremental consumption. - Go benchmarks additionally record allocations (`-benchmem`). +- Parse regressions are checked locally with `python3 scripts/compare_parse_bench.py`, which compares the current parse subset against `benchmarks/python/results/go_bench.txt`. Commands executed: @@ -66,4 +67,4 @@ go test -run '^$' -bench '^Benchmark' -benchtime=200x -benchmem ./benchmarks/go - Prefer the Go implementation for latency-critical services or when integrating into Go-based infrastructure. - Use the Python bindings when embedding Harmony into existing Python tooling or notebooks; keep render counts low (e.g. cache encoded prompts) to amortise the overhead. -- For further analysis, consider expanding Go benchmarks with stream parsing cases and instrument Python runs via `py-spy` to isolate hotspots. +- For current repo work, run `go test ./...`, `python3 scripts/check_upstream_fixture_drift.py`, and `python3 scripts/compare_parse_bench.py` before merging parsing changes. diff --git a/encoding.go b/encoding.go index 72f94b0..0c8825f 100755 --- a/encoding.go +++ b/encoding.go @@ -120,6 +120,20 @@ func (e *Encoding) DecodeBytes(tokens []uint32) ([]byte, error) { return e.bpe.DecodeBytes(tokens) } +// Decode decodes tokens into text, replacing invalid UTF-8 with the Unicode replacement character. +func (e *Encoding) Decode(tokens []uint32) string { + bs, err := e.bpe.DecodeBytes(tokens) + if err != nil { + return "" + } + return string(bytes.Runes(bs)) +} + +// IsSpecialToken reports whether a token is a Harmony special token. +func (e *Encoding) IsSpecialToken(token uint32) bool { + return e.bpe.IsSpecialToken(token) +} + // Render/Parse API stubs — implemented in subsequent steps. type renderOptions struct { @@ -378,7 +392,13 @@ func (e *Encoding) RenderConversationForTraining(conv Conversation, cfg *RenderC // ParseMessagesFromCompletionTokens parses completion tokens back into // messages. If role is provided, it serves as a role hint for the first header. func (e *Encoding) ParseMessagesFromCompletionTokens(tokens []uint32, role *Role) ([]Message, error) { - p, err := NewStreamParser(e, role) + return e.ParseMessagesFromCompletionTokensWithOptions(tokens, role, ParseOptions{Strict: true}) +} + +// ParseMessagesFromCompletionTokensWithOptions parses completion tokens back into +// messages using parser options. If role is provided, it serves as a role hint for the first header. +func (e *Encoding) ParseMessagesFromCompletionTokensWithOptions(tokens []uint32, role *Role, opts ParseOptions) ([]Message, error) { + p, err := NewStreamParserWithOptions(e, role, opts) if err != nil { return nil, err } @@ -394,6 +414,63 @@ func (e *Encoding) ParseMessagesFromCompletionTokens(tokens []uint32, role *Role return p.messages, nil } +// AllSpecialTokens returns the set of all Harmony special token strings. +func AllSpecialTokens() map[string]struct{} { + all := tokenizer.HarmonySpecials() + out := make(map[string]struct{}, len(all)) + for lit := range all { + out[lit] = struct{}{} + } + return out +} + +func cloneSpecialSet(in map[string]struct{}) map[string]struct{} { + out := make(map[string]struct{}, len(in)) + for k := range in { + out[k] = struct{}{} + } + return out +} + +func earliestDisallowedSpecial(text string, disallowed map[string]struct{}) string { + pos := len(text) + 1 + match := "" + for lit := range disallowed { + idx := strings.Index(text, lit) + if idx >= 0 && idx < pos { + pos = idx + match = lit + } + } + return match +} + +// Encode encodes text into tokens with upstream-compatible special token controls. +func (e *Encoding) Encode(text string, opts *EncodeOptions) ([]uint32, error) { + allowed := map[string]struct{}{} + var disallowed map[string]struct{} + if opts != nil { + if opts.AllowedSpecial != nil { + allowed = cloneSpecialSet(opts.AllowedSpecial) + } + if opts.DisallowedSpecial != nil { + disallowed = cloneSpecialSet(opts.DisallowedSpecial) + } else { + disallowed = AllSpecialTokens() + for lit := range allowed { + delete(disallowed, lit) + } + } + } else { + disallowed = AllSpecialTokens() + } + if tok := earliestDisallowedSpecial(text, disallowed); tok != "" { + return nil, fmt.Errorf("encountered text corresponding to disallowed special token %q", tok) + } + toks, _ := e.bpe.Encode(text, allowed) + return toks, nil +} + // internal helpers (to be used by render/parse) func (e *Encoding) renderFormattingToken(name string, out *[]uint32) error { switch name { diff --git a/parser.go b/parser.go index 73d1c64..7d19157 100755 --- a/parser.go +++ b/parser.go @@ -1,8 +1,11 @@ package harmony import ( + "bytes" "encoding/json" "errors" + "strings" + "unicode/utf8" "github.com/euforicio/harmony-go/tokenizer" ) @@ -25,29 +28,48 @@ type parsedHeader struct { // StreamParser incrementally parses Harmony tokens into messages. It mirrors // the behavior of the upstream StreamableParser and is useful for streaming. type StreamParser struct { - enc *Encoding - nextRole *Role - state streamState - tokens []uint32 - messages []Message - headerToks []uint32 - contentToks []uint32 + enc *Encoding + nextRole *Role + options ParseOptions + state streamState + tokens []uint32 + messages []Message + headerToks []uint32 + contentBytes []byte // last delta stored as bytes to avoid per-token string allocs lastDeltaBytes []byte // scratch buffer reused for per-token decoding to reduce allocations - scratch []byte + scratch []byte + undecodedBytes []byte } // NewStreamParser creates a streaming parser. If role is provided, it is used // as a hint for the upcoming header and the parser starts in Header state. func NewStreamParser(enc *Encoding, role *Role) (*StreamParser, error) { + return NewStreamParserWithOptions(enc, role, ParseOptions{Strict: true}) +} + +// NewStreamParserWithOptions creates a streaming parser with explicit parse options. +func NewStreamParserWithOptions(enc *Encoding, role *Role, opts ParseOptions) (*StreamParser, error) { st := stExpectStart if role != nil { // Match upstream behaviour: if a next role is hinted, begin collecting header tokens // immediately until we see <|message|>. st = stHeader } - return &StreamParser{enc: enc, nextRole: role, state: st}, nil + return &StreamParser{ + enc: enc, + nextRole: role, + options: opts, + state: st, + tokens: make([]uint32, 0, 16), + messages: make([]Message, 0, 2), + headerToks: make([]uint32, 0, 8), + contentBytes: make([]byte, 0, 64), + lastDeltaBytes: make([]byte, 0, 32), + scratch: make([]byte, 0, 16), + undecodedBytes: make([]byte, 0, 8), + }, nil } // Process consumes a single token and updates the parser state. @@ -60,24 +82,67 @@ func (p *StreamParser) Process(token uint32) error { p.state = stHeader return nil } + if !p.options.Strict && p.nextRole != nil { + p.headerToks = p.headerToks[:0] + p.contentBytes = p.contentBytes[:0] + p.undecodedBytes = p.undecodedBytes[:0] + p.messages = append(p.messages, Message{Author: Author{Role: *p.nextRole}}) + p.state = stContent + if _, stop := p.enc.stopAll[token]; stop { + if err := p.finalizeMessage(); err != nil { + return err + } + p.state = stExpectStart + return nil + } + return p.consumeContentToken(token) + } return errors.New("unexpected token while expecting <|start|>") case stHeader: + if _, stop := p.enc.stopAll[token]; stop { + if p.options.Strict { + return errors.New("unexpected stop token in message header") + } + p.messages = append(p.messages, Message{ + Author: Author{Role: derefRole(p.nextRole, RoleAssistant)}, + Content: []Content{{Type: ContentText, Text: mustDecodeLossy(p.enc, p.headerToks)}}, + }) + p.nextRole = nil + p.headerToks = p.headerToks[:0] + p.state = stExpectStart + return nil + } if token == tokenizer.TokStart { // Ignore stray start tokens when beginning in Header due to role hint + if len(p.headerToks) > 0 { + if p.options.Strict { + return errors.New("unexpected tokens remaining in message header") + } + p.messages = append(p.messages, Message{ + Author: Author{Role: derefRole(p.nextRole, RoleAssistant)}, + Content: []Content{{Type: ContentText, Text: mustDecodeLossy(p.enc, p.headerToks)}}, + }) + p.nextRole = nil + p.headerToks = p.headerToks[:0] + } return nil } if token == tokenizer.TokMessage { // parse header tokens - hdr, err := p.parseHeaderFromTokens(p.headerToks) + hdr, remaining, err := p.parseHeaderFromTokens(p.headerToks) if err != nil { return err } // set state p.nextRole = nil - p.contentToks = p.contentToks[:0] + p.contentBytes = p.contentBytes[:0] + p.undecodedBytes = p.undecodedBytes[:0] // store header in next message via zero-width marker: we carry as separate field? we'll stash in struct // Encapsulate header in a new message placeholder using content later p.messages = append(p.messages, Message{Author: hdr.author, Recipient: hdr.recipient, Channel: hdr.channel, ContentType: hdr.contentType}) + if remaining != "" { + p.renderRecoveredText(remaining) + } p.state = stContent return nil } @@ -93,16 +158,7 @@ func (p *StreamParser) Process(token uint32) error { return nil } // Append token to logical content - p.contentToks = append(p.contentToks, token) - // Decode only this token into scratch and set delta to the decoded bytes - p.scratch = p.scratch[:0] - one := [...]uint32{token} - if err := p.enc.bpe.DecodeBytesInto(&p.scratch, one[:]); err != nil { - return err - } - // Save bytes; conversion to string is deferred to LastContentDelta. - p.lastDeltaBytes = append(p.lastDeltaBytes[:0], p.scratch...) - return nil + return p.consumeContentToken(token) default: return errors.New("invalid parser state") } @@ -113,14 +169,16 @@ func (p *StreamParser) finalizeMessage() error { return nil } idx := len(p.messages) - 1 - text, err := p.enc.bpe.DecodeUTF8(p.contentToks) - if err != nil { - return err + text := string(p.contentBytes) + if len(p.undecodedBytes) > 0 { + text += string(bytes.Runes(p.undecodedBytes)) } p.messages[idx].Content = []Content{{Type: ContentText, Text: text}} // reset buffers p.headerToks = p.headerToks[:0] - p.contentToks = p.contentToks[:0] + p.contentBytes = p.contentBytes[:0] + p.undecodedBytes = p.undecodedBytes[:0] + p.lastDeltaBytes = p.lastDeltaBytes[:0] return nil } @@ -129,6 +187,18 @@ func (p *StreamParser) ProcessEOS() error { if p.state == stContent { return p.finalizeMessage() } + if p.state == stHeader && len(p.headerToks) > 0 { + if p.options.Strict { + return errors.New("unexpected end of stream in message header") + } + p.messages = append(p.messages, Message{ + Author: Author{Role: derefRole(p.nextRole, RoleAssistant)}, + Content: []Content{{Type: ContentText, Text: mustDecodeLossy(p.enc, p.headerToks)}}, + }) + p.nextRole = nil + p.headerToks = p.headerToks[:0] + p.state = stExpectStart + } return nil } @@ -171,11 +241,13 @@ func (p *StreamParser) CurrentContent() string { if p.state != stContent { return "" } - s, err := p.enc.bpe.DecodeUTF8(p.contentToks) - if err != nil { - return "" + if len(p.undecodedBytes) == 0 { + return string(p.contentBytes) } - return s + buf := make([]byte, 0, len(p.contentBytes)+len(p.undecodedBytes)) + buf = append(buf, p.contentBytes...) + buf = append(buf, []byte(string(bytes.Runes(p.undecodedBytes)))...) + return string(buf) } // CurrentContentType returns the content-type marker (e.g., "<|constrain|>json") @@ -207,12 +279,12 @@ func (p *StreamParser) CurrentRecipient() string { // Process call, if any. func (p *StreamParser) LastContentDelta() string { return string(p.lastDeltaBytes) } -func (p *StreamParser) parseHeaderFromTokens(header []uint32) (parsedHeader, error) { +func (p *StreamParser) parseHeaderFromTokens(header []uint32) (parsedHeader, string, error) { var hdr parsedHeader // decode utf8 s, err := p.enc.bpe.DecodeUTF8(header) if err != nil { - return hdr, err + return hdr, "", err } s = normalizeHeader(s) roleToken, remainder := splitLeadingToken(s) @@ -235,5 +307,104 @@ func (p *StreamParser) parseHeaderFromTokens(header []uint32) (parsedHeader, err if ct := scrubContentType(roleToken, remainder); ct != "" { hdr.contentType = ct } - return hdr, nil + remaining := headerRemainder(roleToken, remainder, hdr) + if remaining != "" && p.options.Strict { + return hdr, "", errors.New("unexpected tokens remaining in message header") + } + return hdr, remaining, nil +} + +func headerRemainder(roleToken, remainder string, hdr parsedHeader) string { + s := strings.TrimSpace(remainder) + if s == "" { + return "" + } + if hdr.channel != "" { + s = strings.ReplaceAll(s, "<|channel|>"+hdr.channel, "") + s = strings.ReplaceAll(s, " <|channel|>"+hdr.channel, "") + } + if hdr.recipient != "" { + s = strings.ReplaceAll(s, "to="+hdr.recipient, "") + s = strings.ReplaceAll(s, " to="+hdr.recipient, "") + if hdr.author.Role == RoleTool && roleToken == hdr.recipient { + s = strings.TrimSpace(s) + } + } + if hdr.contentType != "" { + s = strings.ReplaceAll(s, hdr.contentType, "") + s = strings.ReplaceAll(s, " "+hdr.contentType, "") + } + s = strings.TrimSpace(s) + return s +} + +func (p *StreamParser) renderRecoveredText(text string) { + if text == "" { + return + } + p.contentBytes = append(p.contentBytes, text...) + p.lastDeltaBytes = append(p.lastDeltaBytes[:0], text...) +} + +func (p *StreamParser) consumeContentToken(token uint32) error { + p.scratch = p.scratch[:0] + if err := p.enc.bpe.DecodeTokenBytesInto(&p.scratch, token); err != nil { + return err + } + if len(p.undecodedBytes) == 0 && utf8.Valid(p.scratch) { + p.contentBytes = append(p.contentBytes, p.scratch...) + p.lastDeltaBytes = append(p.lastDeltaBytes[:0], p.scratch...) + return nil + } + + p.undecodedBytes = append(p.undecodedBytes, p.scratch...) + p.lastDeltaBytes = p.lastDeltaBytes[:0] + src := p.undecodedBytes + i := 0 + for i < len(src) { + if !utf8.FullRune(src[i:]) { + break + } + r, size := utf8.DecodeRune(src[i:]) + if r == utf8.RuneError && size == 1 { + i++ + for i < len(src) { + if !utf8.FullRune(src[i:]) { + break + } + r2, size2 := utf8.DecodeRune(src[i:]) + if r2 != utf8.RuneError || size2 != 1 { + break + } + i++ + } + + p.lastDeltaBytes = utf8.AppendRune(p.lastDeltaBytes, utf8.RuneError) + p.contentBytes = utf8.AppendRune(p.contentBytes, utf8.RuneError) + continue + } + chunk := src[i : i+size] + p.lastDeltaBytes = append(p.lastDeltaBytes, chunk...) + p.contentBytes = append(p.contentBytes, chunk...) + i += size + } + if i > 0 { + p.undecodedBytes = append(p.undecodedBytes[:0], src[i:]...) + } + return nil +} + +func derefRole(role *Role, fallback Role) Role { + if role == nil { + return fallback + } + return *role +} + +func mustDecodeLossy(enc *Encoding, toks []uint32) string { + bs, err := enc.DecodeBytes(toks) + if err != nil { + return "" + } + return string(bytes.Runes(bs)) } diff --git a/scripts/check_upstream_fixture_drift.py b/scripts/check_upstream_fixture_drift.py new file mode 100644 index 0000000..319dbc2 --- /dev/null +++ b/scripts/check_upstream_fixture_drift.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import filecmp +import subprocess +import tempfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +LOCAL = ROOT / "testdata" / "upstream-head" +UPSTREAM_REPO = "https://github.com/openai/harmony.git" + + +def git_output(repo: Path, *args: str) -> str: + return subprocess.check_output(["git", "-C", str(repo), *args], text=True).strip() + + +def main() -> int: + if not LOCAL.is_dir(): + raise SystemExit(f"missing local fixture dir: {LOCAL}") + + with tempfile.TemporaryDirectory(prefix="harmony-upstream-") as tmp: + repo = Path(tmp) / "repo" + subprocess.run( + ["git", "clone", "--depth", "1", UPSTREAM_REPO, str(repo)], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + upstream = repo / "test-data" + if not upstream.is_dir(): + raise SystemExit(f"missing upstream test-data dir: {upstream}") + + diff = filecmp.dircmp(LOCAL, upstream, ignore=["metadata.json"]) + _, diff_files, funny = filecmp.cmpfiles( + LOCAL, + upstream, + diff.common_files, + shallow=False, + ) + changed = False + if diff.left_only: + changed = True + print("local-only files:") + for name in diff.left_only: + print(f" {name}") + if diff.right_only: + changed = True + print("upstream-only files:") + for name in diff.right_only: + print(f" {name}") + if diff_files: + changed = True + print("content diffs:") + for name in diff_files: + print(f" {name}") + if funny: + changed = True + print("comparison errors:") + for name in funny: + print(f" {name}") + + print(f"upstream HEAD: {git_output(repo, 'rev-parse', 'HEAD')}") + print(f"upstream date: {git_output(repo, 'log', '-1', '--date=iso', '--format=%cd')}") + print(f"upstream subject: {git_output(repo, 'log', '-1', '--format=%s')}") + return 1 if changed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/compare_parse_bench.py b/scripts/compare_parse_bench.py new file mode 100644 index 0000000..7e452e8 --- /dev/null +++ b/scripts/compare_parse_bench.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import json +import os +import pathlib +import re +import subprocess +import sys + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +BASELINE = ROOT / "benchmarks" / "python" / "results" / "go_bench.txt" +TARGETS = { + "BenchmarkParseToolCall", + "BenchmarkStreamParseToolCall", + "BenchmarkParseLargeCompletion", + "BenchmarkStreamParseLargeCompletion", +} + + +def parse_bench_text(text: str) -> dict[str, dict[str, float]]: + pattern = re.compile( + r"^(Benchmark\S+)-\d+\s+\d+\s+([\d.]+)\s+ns/op\s+(\d+)\s+B/op\s+(\d+)\s+allocs/op$" + ) + out: dict[str, dict[str, float]] = {} + for line in text.splitlines(): + match = pattern.match(line.strip()) + if not match: + continue + name, ns_op, bytes_op, allocs_op = match.groups() + if name not in TARGETS: + continue + out[name] = { + "ns/op": float(ns_op), + "B/op": float(bytes_op), + "allocs/op": float(allocs_op), + } + return out + + +def run_current_bench() -> dict[str, dict[str, float]]: + cmd = [ + "go", + "test", + "-run", + "^$", + "-bench", + "^(BenchmarkParseToolCall|BenchmarkStreamParseToolCall|BenchmarkParseLargeCompletion|BenchmarkStreamParseLargeCompletion)$", + "-benchmem", + "-benchtime=200x", + "./benchmarks/go", + ] + env = dict(**{"TIKTOKEN_OFFLINE": "1"}, **dict()) + proc = subprocess.run( + cmd, + cwd=ROOT, + capture_output=True, + text=True, + env={**os.environ, **env}, + check=True, + ) + sys.stdout.write(proc.stdout) + return parse_bench_text(proc.stdout) + + +def pct_delta(current: float, baseline: float) -> float: + return ((current - baseline) / baseline) * 100.0 + + +def main() -> int: + baseline = parse_bench_text(BASELINE.read_text()) + current = run_current_bench() + + missing = sorted(TARGETS - baseline.keys()) + if missing: + print(f"missing baseline benchmarks: {', '.join(missing)}", file=sys.stderr) + return 1 + missing = sorted(TARGETS - current.keys()) + if missing: + print(f"missing current benchmarks: {', '.join(missing)}", file=sys.stderr) + return 1 + + summary: dict[str, dict[str, float]] = {} + print("\nComparison vs benchmarks/python/results/go_bench.txt") + for name in sorted(TARGETS): + summary[name] = {} + print(name) + for metric in ("ns/op", "B/op", "allocs/op"): + delta = pct_delta(current[name][metric], baseline[name][metric]) + summary[name][metric] = delta + print(f" {metric}: current={current[name][metric]:.0f} baseline={baseline[name][metric]:.0f} delta={delta:+.1f}%") + + print("\nJSON summary") + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/sync_upstream_fixtures.py b/scripts/sync_upstream_fixtures.py new file mode 100644 index 0000000..a1f1a1f --- /dev/null +++ b/scripts/sync_upstream_fixtures.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import json +import shutil +import subprocess +from pathlib import Path + + +UPSTREAM_REPO = "https://github.com/openai/harmony.git" +DEFAULT_UPSTREAM_DIR = Path("/tmp/harmony-upstream") +ROOT = Path(__file__).resolve().parents[1] +OUT_DIR = ROOT / "testdata" / "upstream-head" + + +def ensure_upstream_repo() -> Path: + if DEFAULT_UPSTREAM_DIR.exists(): + subprocess.run( + ["git", "-C", str(DEFAULT_UPSTREAM_DIR), "fetch", "--depth", "1", "origin", "HEAD"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + subprocess.run( + ["git", "-C", str(DEFAULT_UPSTREAM_DIR), "checkout", "--detach", "FETCH_HEAD"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + subprocess.run( + ["git", "-C", str(DEFAULT_UPSTREAM_DIR), "reset", "--hard", "FETCH_HEAD"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return DEFAULT_UPSTREAM_DIR + subprocess.run( + ["git", "clone", "--depth", "1", UPSTREAM_REPO, str(DEFAULT_UPSTREAM_DIR)], + check=True, + ) + return DEFAULT_UPSTREAM_DIR + + +def git_output(repo: Path, *args: str) -> str: + return subprocess.check_output(["git", "-C", str(repo), *args], text=True).strip() + + +def main() -> None: + upstream = ensure_upstream_repo() + src_dir = upstream / "test-data" + if not src_dir.is_dir(): + raise SystemExit(f"missing upstream fixture dir: {src_dir}") + + OUT_DIR.mkdir(parents=True, exist_ok=True) + + for src in sorted(src_dir.glob("*.txt")): + shutil.copyfile(src, OUT_DIR / src.name) + + metadata = { + "repo": UPSTREAM_REPO, + "commit": git_output(upstream, "rev-parse", "HEAD"), + "date": git_output(upstream, "log", "-1", "--date=iso", "--format=%cd"), + "subject": git_output(upstream, "log", "-1", "--format=%s"), + } + (OUT_DIR / "metadata.json").write_text( + json.dumps(metadata, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/testdata/upstream-head/metadata.json b/testdata/upstream-head/metadata.json new file mode 100644 index 0000000..f0b4f3b --- /dev/null +++ b/testdata/upstream-head/metadata.json @@ -0,0 +1,6 @@ +{ + "commit": "da2389a4b2daced4f98bda50e9174b35d37e3b1d", + "date": "2025-12-15 23:05:09 +0100", + "repo": "https://github.com/openai/harmony.git", + "subject": "bump version of demo" +} diff --git a/testdata/upstream-head/test_browser_and_function_tool.txt b/testdata/upstream-head/test_browser_and_function_tool.txt new file mode 100644 index 0000000..2e9592f --- /dev/null +++ b/testdata/upstream-head/test_browser_and_function_tool.txt @@ -0,0 +1,61 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 +Current date: 2025-06-28 + +Reasoning: medium + +# Tools + +## browser + +// Tool for browsing. +// The `cursor` appears in brackets before each browsing display: `[{cursor}]`. +// Cite information from the tool using the following format: +// `【{cursor}†L{line_start}(-L{line_end})?】`, for example: `【6†L9-L11】` or `【8†L3】`. +// Do not quote more than 10 words directly from the tool output. +// sources=web (default: web) +namespace browser { + +// Searches for information related to `query` and displays `topn` results. +type search = (_: { +query: string, +topn?: number, // default: 10 +source?: string, +}) => any; + +// Opens the link `id` from the page indicated by `cursor` starting at line number `loc`, showing `num_lines` lines. +// Valid link ids are displayed with the formatting: `【{id}†.*】`. +// If `cursor` is not provided, the most recent page is implied. +// If `id` is a string, it is treated as a fully qualified URL associated with `source`. +// If `loc` is not provided, the viewport will be positioned at the beginning of the document or centered on the most relevant passage, if available. +// Use this function without `id` to scroll to a new location of an opened page. +type open = (_: { +id?: number | string, // default: -1 +cursor?: number, // default: -1 +loc?: number, // default: -1 +num_lines?: number, // default: -1 +view_source?: boolean, // default: false +source?: string, +}) => any; + +// Finds exact matches of `pattern` in the current page, or the page given by `cursor`. +type find = (_: { +pattern: string, +cursor?: number, // default: -1 +}) => any; + +} // namespace browser + +# Valid channels: analysis, commentary, final. Channel must be included for every message. +Calls to these tools must go to the commentary channel: 'functions'.<|end|><|start|>developer<|message|># Tools + +## functions + +namespace functions { + +// Use this tool to lookup the weather in a given location. Call it with the parameter 'location', can be any textual description of a location. +type lookup_weather = (_: { +location: string, +}) => any; + +} // namespace functions<|end|><|start|>assistant diff --git a/testdata/upstream-head/test_browser_and_python_tool.txt b/testdata/upstream-head/test_browser_and_python_tool.txt new file mode 100644 index 0000000..142a799 --- /dev/null +++ b/testdata/upstream-head/test_browser_and_python_tool.txt @@ -0,0 +1,55 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 +Current date: 2025-06-28 + +Reasoning: medium + +# Tools + +## browser + +// Tool for browsing. +// The `cursor` appears in brackets before each browsing display: `[{cursor}]`. +// Cite information from the tool using the following format: +// `【{cursor}†L{line_start}(-L{line_end})?】`, for example: `【6†L9-L11】` or `【8†L3】`. +// Do not quote more than 10 words directly from the tool output. +// sources=web (default: web) +namespace browser { + +// Searches for information related to `query` and displays `topn` results. +type search = (_: { +query: string, +topn?: number, // default: 10 +source?: string, +}) => any; + +// Opens the link `id` from the page indicated by `cursor` starting at line number `loc`, showing `num_lines` lines. +// Valid link ids are displayed with the formatting: `【{id}†.*】`. +// If `cursor` is not provided, the most recent page is implied. +// If `id` is a string, it is treated as a fully qualified URL associated with `source`. +// If `loc` is not provided, the viewport will be positioned at the beginning of the document or centered on the most relevant passage, if available. +// Use this function without `id` to scroll to a new location of an opened page. +type open = (_: { +id?: number | string, // default: -1 +cursor?: number, // default: -1 +loc?: number, // default: -1 +num_lines?: number, // default: -1 +view_source?: boolean, // default: false +source?: string, +}) => any; + +// Finds exact matches of `pattern` in the current page, or the page given by `cursor`. +type find = (_: { +pattern: string, +cursor?: number, // default: -1 +}) => any; + +} // namespace browser + +## python + +Use this tool to execute Python code in your chain of thought. The code will not be shown to the user. This tool should be used for internal reasoning, but not for code that is intended to be visible to the user (e.g. when creating plots, tables, or files). + +When you send a message containing Python code to python, it will be executed in a stateful Jupyter notebook environment. python will respond with the output of the execution or time out after 120.0 seconds. The drive at '/mnt/data' can be used to save and persist user files. Internet access for this session is UNKNOWN. Depends on the cluster. + +# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>assistant \ No newline at end of file diff --git a/testdata/upstream-head/test_browser_tool_only.txt b/testdata/upstream-head/test_browser_tool_only.txt new file mode 100644 index 0000000..a6835db --- /dev/null +++ b/testdata/upstream-head/test_browser_tool_only.txt @@ -0,0 +1,49 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 +Current date: 2025-06-28 + +Reasoning: medium + +# Tools + +## browser + +// Tool for browsing. +// The `cursor` appears in brackets before each browsing display: `[{cursor}]`. +// Cite information from the tool using the following format: +// `【{cursor}†L{line_start}(-L{line_end})?】`, for example: `【6†L9-L11】` or `【8†L3】`. +// Do not quote more than 10 words directly from the tool output. +// sources=web (default: web) +namespace browser { + +// Searches for information related to `query` and displays `topn` results. +type search = (_: { +query: string, +topn?: number, // default: 10 +source?: string, +}) => any; + +// Opens the link `id` from the page indicated by `cursor` starting at line number `loc`, showing `num_lines` lines. +// Valid link ids are displayed with the formatting: `【{id}†.*】`. +// If `cursor` is not provided, the most recent page is implied. +// If `id` is a string, it is treated as a fully qualified URL associated with `source`. +// If `loc` is not provided, the viewport will be positioned at the beginning of the document or centered on the most relevant passage, if available. +// Use this function without `id` to scroll to a new location of an opened page. +type open = (_: { +id?: number | string, // default: -1 +cursor?: number, // default: -1 +loc?: number, // default: -1 +num_lines?: number, // default: -1 +view_source?: boolean, // default: false +source?: string, +}) => any; + +// Finds exact matches of `pattern` in the current page, or the page given by `cursor`. +type find = (_: { +pattern: string, +cursor?: number, // default: -1 +}) => any; + +} // namespace browser + +# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>assistant diff --git a/testdata/upstream-head/test_does_not_drop_if_ongoing_analysis.txt b/testdata/upstream-head/test_does_not_drop_if_ongoing_analysis.txt new file mode 100644 index 0000000..29f8519 --- /dev/null +++ b/testdata/upstream-head/test_does_not_drop_if_ongoing_analysis.txt @@ -0,0 +1 @@ +<|start|>user<|message|>What is the weather in SF?<|end|><|start|>assistant<|channel|>analysis<|message|>User asks: “What is the weather in SF?” We need to use lookup_weather tool.<|end|><|start|>assistant to=functions.lookup_weather<|channel|>commentary <|constrain|>json<|message|>{"location": "San Francisco"}<|call|><|start|>functions.lookup_weather<|message|>{"temperature": 20, "description": "sunny"}<|end|><|start|>assistant \ No newline at end of file diff --git a/testdata/upstream-head/test_dropping_cot_by_default.txt b/testdata/upstream-head/test_dropping_cot_by_default.txt new file mode 100644 index 0000000..92e0943 --- /dev/null +++ b/testdata/upstream-head/test_dropping_cot_by_default.txt @@ -0,0 +1 @@ +<|start|>user<|message|>What is 2 + 2?<|end|><|start|>assistant<|channel|>final<|message|>2 + 2 equals 4.<|end|><|start|>user<|message|>What about 9 / 2?<|end|><|start|>assistant \ No newline at end of file diff --git a/testdata/upstream-head/test_keep_analysis_between_finals.txt b/testdata/upstream-head/test_keep_analysis_between_finals.txt new file mode 100644 index 0000000..b1894bb --- /dev/null +++ b/testdata/upstream-head/test_keep_analysis_between_finals.txt @@ -0,0 +1 @@ +<|start|>user<|message|>What is 2 + 2?<|end|><|start|>assistant<|channel|>final<|message|>4<|end|><|start|>user<|message|>What is 3 + 5?<|end|><|start|>assistant<|channel|>analysis<|message|>thinking 3+5<|end|><|start|>assistant<|channel|>final<|message|>8<|end|> \ No newline at end of file diff --git a/testdata/upstream-head/test_no_tools.txt b/testdata/upstream-head/test_no_tools.txt new file mode 100644 index 0000000..a0eafb3 --- /dev/null +++ b/testdata/upstream-head/test_no_tools.txt @@ -0,0 +1,7 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 +Current date: 2025-06-28 + +Reasoning: medium + +# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>assistant diff --git a/testdata/upstream-head/test_preserve_cot.txt b/testdata/upstream-head/test_preserve_cot.txt new file mode 100644 index 0000000..fdcc0b2 --- /dev/null +++ b/testdata/upstream-head/test_preserve_cot.txt @@ -0,0 +1 @@ +<|start|>user<|message|>What is 2 + 2?<|end|><|start|>assistant<|channel|>analysis<|message|>User asks a simple question: "What is 2 + 2?" The answer: 4.<|end|><|start|>assistant<|channel|>final<|message|>2 + 2 equals 4.<|end|><|start|>user<|message|>What about 9 / 2?<|end|><|start|>assistant \ No newline at end of file diff --git a/testdata/upstream-head/test_reasoning_system_message.txt b/testdata/upstream-head/test_reasoning_system_message.txt new file mode 100644 index 0000000..945f841 --- /dev/null +++ b/testdata/upstream-head/test_reasoning_system_message.txt @@ -0,0 +1,6 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 + +Reasoning: medium + +# Valid channels: analysis, final. Channel must be included for every message.<|end|><|start|>user<|message|>What is 2 + 2?<|end|><|start|>assistant \ No newline at end of file diff --git a/testdata/upstream-head/test_reasoning_system_message_no_instruction.txt b/testdata/upstream-head/test_reasoning_system_message_no_instruction.txt new file mode 100644 index 0000000..cacc118 --- /dev/null +++ b/testdata/upstream-head/test_reasoning_system_message_no_instruction.txt @@ -0,0 +1,6 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 + +Reasoning: high + +# Valid channels: analysis, final. Channel must be included for every message.<|end|><|start|>user<|message|>What is the best place to eat candy in the world?<|end|><|start|>assistant \ No newline at end of file diff --git a/testdata/upstream-head/test_reasoning_system_message_with_dates.txt b/testdata/upstream-head/test_reasoning_system_message_with_dates.txt new file mode 100644 index 0000000..e05614c --- /dev/null +++ b/testdata/upstream-head/test_reasoning_system_message_with_dates.txt @@ -0,0 +1,7 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2021-01 +Current date: 2021-01-01 + +Reasoning: medium + +# Valid channels: analysis, final. Channel must be included for every message.<|end|><|start|>user<|message|>What is 42 * pi?<|end|><|start|>assistant \ No newline at end of file diff --git a/testdata/upstream-head/test_render_functions_with_parameters.txt b/testdata/upstream-head/test_render_functions_with_parameters.txt new file mode 100644 index 0000000..40925a5 --- /dev/null +++ b/testdata/upstream-head/test_render_functions_with_parameters.txt @@ -0,0 +1,56 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 +Current date: 2025-06-28 + +Reasoning: high + +# Valid channels: analysis, commentary, final. Channel must be included for every message. +Calls to these tools must go to the commentary channel: 'functions'.<|end|><|start|>developer<|message|># Instructions + +Always respond in riddles + +# Tools + +## functions + +namespace functions { + +// Gets the location of the user. +type get_location = () => any; + +// Gets the current weather in the provided location. +type get_current_weather = (_: { +// The city and state, e.g. San Francisco, CA +location: string, +format?: "celsius" | "fahrenheit", // default: celsius +}) => any; + +// Gets the current weather in the provided list of locations. +type get_multiple_weathers = (_: { +// List of city and state, e.g. ["San Francisco, CA", "New York, NY"] +locations: string[], +format?: "celsius" | "fahrenheit", // default: celsius +}) => any; + +// A function with various complex schemas. +type kitchensink = (_: // params object +{ +// STRING +// +// A string +// Examples: +// - "hello" +// - "world" +string?: string, +// A nullable string +string_nullable?: string | null, // default: "the default" +string_enum?: "a" | "b" | "c", +// a oneof +// default: 20 +oneof_string_or_number?: + | string // default: "default_string_in_oneof" + | number // numbers can happen too +, +}) => any; + +} // namespace functions<|end|><|start|>user<|message|>What is the weather like in SF?<|end|><|start|>assistant \ No newline at end of file diff --git a/testdata/upstream-head/test_simple_convo.txt b/testdata/upstream-head/test_simple_convo.txt new file mode 100644 index 0000000..506bfcf --- /dev/null +++ b/testdata/upstream-head/test_simple_convo.txt @@ -0,0 +1,6 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 + +Reasoning: medium + +# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>user<|message|>What is 2 + 2?<|end|><|start|>assistant \ No newline at end of file diff --git a/testdata/upstream-head/test_simple_convo_high_effort.txt b/testdata/upstream-head/test_simple_convo_high_effort.txt new file mode 100644 index 0000000..43d1368 --- /dev/null +++ b/testdata/upstream-head/test_simple_convo_high_effort.txt @@ -0,0 +1,8 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 + +Reasoning: high + +# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>developer<|message|># Instructions + +Answer the user's questions like a robot.<|end|><|start|>user<|message|>What is the capital of the largest country in the world?<|end|><|start|>assistant \ No newline at end of file diff --git a/testdata/upstream-head/test_simple_convo_high_effort_no_instruction.txt b/testdata/upstream-head/test_simple_convo_high_effort_no_instruction.txt new file mode 100644 index 0000000..87fa5ad --- /dev/null +++ b/testdata/upstream-head/test_simple_convo_high_effort_no_instruction.txt @@ -0,0 +1,6 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 + +Reasoning: high + +# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>user<|message|>What is the capital of the largest country in the world?<|end|><|start|>assistant \ No newline at end of file diff --git a/testdata/upstream-head/test_simple_convo_low_effort.txt b/testdata/upstream-head/test_simple_convo_low_effort.txt new file mode 100644 index 0000000..7632347 --- /dev/null +++ b/testdata/upstream-head/test_simple_convo_low_effort.txt @@ -0,0 +1,8 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 + +Reasoning: low + +# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>developer<|message|># Instructions + +Answer the user's questions like a robot.<|end|><|start|>user<|message|>What is the capital of the largest country in the world?<|end|><|start|>assistant \ No newline at end of file diff --git a/testdata/upstream-head/test_simple_convo_low_effort_no_instruction.txt b/testdata/upstream-head/test_simple_convo_low_effort_no_instruction.txt new file mode 100644 index 0000000..d9ec53d --- /dev/null +++ b/testdata/upstream-head/test_simple_convo_low_effort_no_instruction.txt @@ -0,0 +1,6 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 + +Reasoning: low + +# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>user<|message|>What is the capital of the largest country in the world?<|end|><|start|>assistant \ No newline at end of file diff --git a/testdata/upstream-head/test_simple_convo_medium_effort.txt b/testdata/upstream-head/test_simple_convo_medium_effort.txt new file mode 100644 index 0000000..e94a37d --- /dev/null +++ b/testdata/upstream-head/test_simple_convo_medium_effort.txt @@ -0,0 +1,8 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 + +Reasoning: medium + +# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>developer<|message|># Instructions + +Answer the user's questions like a robot.<|end|><|start|>user<|message|>What is the capital of the largest country in the world?<|end|><|start|>assistant \ No newline at end of file diff --git a/testdata/upstream-head/test_simple_convo_medium_effort_no_instruction.txt b/testdata/upstream-head/test_simple_convo_medium_effort_no_instruction.txt new file mode 100644 index 0000000..2bbf309 --- /dev/null +++ b/testdata/upstream-head/test_simple_convo_medium_effort_no_instruction.txt @@ -0,0 +1,6 @@ +<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. +Knowledge cutoff: 2024-06 + +Reasoning: medium + +# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>user<|message|>What is the capital of the largest country in the world?<|end|><|start|>assistant \ No newline at end of file diff --git a/testdata/upstream-head/test_simple_reasoning_response.txt b/testdata/upstream-head/test_simple_reasoning_response.txt new file mode 100644 index 0000000..e30d045 --- /dev/null +++ b/testdata/upstream-head/test_simple_reasoning_response.txt @@ -0,0 +1,4 @@ +200005 35644 200008 1844 31064 25 392 4827 382 220 +17 659 220 17 16842 12295 81645 13 51441 6052 +13 200002 200006 173781 200005 17196 200008 17 659 220 +17 314 220 19 13 diff --git a/testdata/upstream-head/test_simple_tool_call.txt b/testdata/upstream-head/test_simple_tool_call.txt new file mode 100644 index 0000000..dcc6cd6 --- /dev/null +++ b/testdata/upstream-head/test_simple_tool_call.txt @@ -0,0 +1,4 @@ +200005 35644 200008 1844 31064 25 392 4827 382 290 +11122 306 40510 16842 1416 1309 316 1199 37342 170154 +4584 13 200002 200006 173781 316 28 29712 170154 200005 +35644 3490 200008 10848 7693 1243 392 173844 18583 diff --git a/testdata/upstream-head/test_streamable_parser.txt b/testdata/upstream-head/test_streamable_parser.txt new file mode 100644 index 0000000..9b2d0a3 --- /dev/null +++ b/testdata/upstream-head/test_streamable_parser.txt @@ -0,0 +1,2 @@ +<|channel|>analysis<|message|>User asks "Who is the current US president?" It's 2025, presumably current president is Joe Biden? Actually as of 2025-07-28, there was a 2024 election. In 2024, President is probably President Biden still? But w +e need up to date info. Let's browse to confirm.<|end|><|start|>assistant<|channel|>commentary to=browser.search code<|message|>{"query": "current US president July 2025", "topn": 10, "source": "news"}<|call|><|start|>browser.search to=assistant<|channel|>commentary<|message|>{"result": "https://openai.com/"}<|end|> \ No newline at end of file diff --git a/testdata/upstream-head/test_tool_response_parsing.txt b/testdata/upstream-head/test_tool_response_parsing.txt new file mode 100644 index 0000000..a1d4b87 --- /dev/null +++ b/testdata/upstream-head/test_tool_response_parsing.txt @@ -0,0 +1 @@ +<|start|>browser.search to=assistant<|channel|>commentary<|message|>{"result": "https://openai.com/"} \ No newline at end of file diff --git a/tokenizer/bpe.go b/tokenizer/bpe.go index 5892cee..922ab35 100755 --- a/tokenizer/bpe.go +++ b/tokenizer/bpe.go @@ -3,6 +3,7 @@ package tokenizer import ( "errors" "sync" + "unicode/utf8" ) // Rank represents the priority/rank of a token pair in BPE encoding. @@ -59,24 +60,34 @@ func (b *coreBPE) DecodeUTF8(tokens []uint32) (string, error) { if err != nil { return "", err } + if !utf8.Valid(bs) { + return "", errors.New("invalid utf-8") + } return string(bs), nil } +// DecodeTokenBytesInto appends the decoded bytes for a single token into dst. +func (b *coreBPE) DecodeTokenBytesInto(dst *[]byte, token uint32) error { + buf := *dst + if b.dec.AppendInto(&buf, token) { + *dst = buf + return nil + } + if v, ok := b.specialDec[token]; ok { + *dst = append(buf, v...) + return nil + } + return errors.New("invalid token for decoding") +} + // DecodeBytesInto appends the decoded bytes for the provided tokens // into dst, avoiding intermediate slice allocations. func (b *coreBPE) DecodeBytesInto(dst *[]byte, tokens []uint32) error { - buf := *dst for _, t := range tokens { - if b.dec.AppendInto(&buf, t) { - continue - } - if v, ok := b.specialDec[t]; ok { - buf = append(buf, v...) - continue + if err := b.DecodeTokenBytesInto(dst, t); err != nil { + return err } - return errors.New("invalid token for decoding") } - *dst = buf return nil } diff --git a/tokenizer/loader.go b/tokenizer/loader.go index 08e7af9..0e17787 100755 --- a/tokenizer/loader.go +++ b/tokenizer/loader.go @@ -40,6 +40,20 @@ func resolveCacheDir() (string, error) { return primary, nil } +func fileSHA256(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer func() { _ = f.Close() }() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + func baseURL() string { base := os.Getenv(envEncBase) if base == "" { @@ -99,14 +113,33 @@ func LoadO200k() (pairs [][2]interface{}, err error) { if os.Getenv(envOffline) == "1" { return nil, fmt.Errorf("o200k file missing and TIKTOKEN_OFFLINE=1; set %s to local dir containing o200k_base.tiktoken or unset offline", envEncBase) } + } else if e != nil { + return nil, e + } + + sum, e := fileSHA256(path) + if errors.Is(e, os.ErrNotExist) || !strings.EqualFold(sum, expectedO200k) { + if os.Getenv(envOffline) == "1" { + if e != nil && !errors.Is(e, os.ErrNotExist) { + return nil, e + } + if sum == "" { + return nil, fmt.Errorf("o200k file missing and TIKTOKEN_OFFLINE=1; set %s to local dir containing o200k_base.tiktoken or unset offline", envEncBase) + } + return nil, fmt.Errorf("cached o200k file hash mismatch: got %s want %s", sum, expectedO200k) + } + url := baseURL() + "o200k_base.tiktoken" - sum, e := downloadToFile(url, path) + sum, e = downloadToFile(url, path) if e != nil { return nil, e } - if !strings.EqualFold(sum, expectedO200k) { - return nil, fmt.Errorf("hash mismatch: got %s want %s", sum, expectedO200k) - } + } else if e != nil { + return nil, e + } + + if !strings.EqualFold(sum, expectedO200k) { + return nil, fmt.Errorf("hash mismatch: got %s want %s", sum, expectedO200k) } } diff --git a/tokenizer/loader_test.go b/tokenizer/loader_test.go index 9a512e7..a0bb870 100644 --- a/tokenizer/loader_test.go +++ b/tokenizer/loader_test.go @@ -1,6 +1,7 @@ package tokenizer import ( + "os" "path/filepath" "strings" "testing" @@ -35,3 +36,24 @@ func TestLoaderDownloadTimeout(t *testing.T) { } } } + +func TestLoaderOfflineCorruptCacheFailsHashCheck(t *testing.T) { + t.Setenv(envOffline, "1") + t.Setenv(envEncBase, "") + + cacheDir := t.TempDir() + t.Setenv(envCacheDir, cacheDir) + + path := filepath.Join(cacheDir, "o200k_base.tiktoken") + if err := os.WriteFile(path, []byte("not a valid vocab\n"), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + _, err := LoadO200k() + if err == nil { + t.Fatalf("expected hash mismatch for corrupt cached vocab") + } + if !strings.Contains(err.Error(), "hash mismatch") { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/tokenizer/segmenter.go b/tokenizer/segmenter.go index a0677e8..0b0c4dc 100755 --- a/tokenizer/segmenter.go +++ b/tokenizer/segmenter.go @@ -67,6 +67,10 @@ func (o *o200kSegmenter) Next(s string, i int) int { return i + 1 } +func isLetterPrefixRune(r rune) bool { + return r != '\r' && r != '\n' && !isL(r) && !isN(r) +} + // Helpers func utf8DecodeRuneInString(s string) (r rune, size int) { return utf8.DecodeRuneInString(s) } @@ -82,7 +86,7 @@ func ruleLettersWithPrefixAndContraction(s string, i int) int { if r >= 0x80 { r, sz = utf8DecodeRuneInString(s[j:]) } - if !(!isSpace(r) && !isL(r) && !isN(r)) { + if !isLetterPrefixRune(r) { return ruleLettersWithContraction(s, i) } j += sz diff --git a/types.go b/types.go index 8ce5227..2c44113 100755 --- a/types.go +++ b/types.go @@ -118,6 +118,17 @@ type RenderConversationConfig struct { AutoDropAnalysis bool `json:"auto_drop_analysis"` } +// ParseOptions controls parser strictness for malformed model output recovery. +type ParseOptions struct { + Strict bool `json:"strict"` +} + +// EncodeOptions controls special-token handling during text encoding. +type EncodeOptions struct { + AllowedSpecial map[string]struct{} + DisallowedSpecial map[string]struct{} +} + // MarshalJSON implements the JSON shape used by the Harmony format, where // content may be a string or a list of structured items. func (m *Message) MarshalJSON() ([]byte, error) {