From 5fc181f8d7b0e357697c55fee56503fea7f6cb79 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Fri, 16 Oct 2026 19:48:03 +0000 Subject: [PATCH 1/5] feat(js): add Python and TypeScript runtime host functions --- .github/workflows/js.yml | 37 +- Cargo.lock | 1 + contracts/capability-parity.json | 48 +- crates/bashkit-js/.gitignore | 6 + crates/bashkit-js/Cargo.toml | 3 +- crates/bashkit-js/README.md | 9 +- .../fixtures/runtime-functions-worker.mjs | 32 ++ .../runtime-compat/runtime-functions.test.mjs | 49 ++ .../__test__/runtime-functions.spec.ts | 512 ++++++++++++++++++ crates/bashkit-js/langchain.ts | 8 +- crates/bashkit-js/package.json | 2 + crates/bashkit-js/runtime-functions.ts | 239 ++++++++ crates/bashkit-js/src/lib.rs | 332 ++++++------ crates/bashkit-js/src/runtime_functions.rs | 167 ++++++ crates/bashkit-js/wrapper.ts | 123 +++-- crates/bashkit/docs/python.md | 2 + crates/bashkit/src/lib.rs | 2 + crates/bashkit/src/runtime_value.rs | 220 ++++++++ crates/bashkit/src/tool.rs | 88 +++ docs/start-node.md | 43 ++ examples/runtime_functions.mjs | 21 + site/src/content/apidocs/typescript.md | 48 +- 22 files changed, 1780 insertions(+), 212 deletions(-) create mode 100644 crates/bashkit-js/__test__/fixtures/runtime-functions-worker.mjs create mode 100644 crates/bashkit-js/__test__/runtime-compat/runtime-functions.test.mjs create mode 100644 crates/bashkit-js/__test__/runtime-functions.spec.ts create mode 100644 crates/bashkit-js/runtime-functions.ts create mode 100644 crates/bashkit-js/src/runtime_functions.rs create mode 100644 crates/bashkit/src/runtime_value.rs create mode 100644 examples/runtime_functions.mjs diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml index 6086e1af8..8915c65a8 100644 --- a/.github/workflows/js.yml +++ b/.github/workflows/js.yml @@ -210,6 +210,7 @@ jobs: ${{ matrix.run }} k8s_orchestrator.mjs ${{ matrix.run }} langchain_integration.mjs ${{ matrix.run }} custom_builtins.mjs + ${{ matrix.run }} runtime_functions.mjs - name: Detect Doppler availability id: doppler @@ -243,11 +244,41 @@ jobs: ${{ matrix.run }} vercel_ai_tool.mjs ${{ matrix.run }} langchain_agent.mjs + callback-platforms: + name: Runtime callbacks (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-latest, windows-latest] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + with: + version: 10.33.0 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v6 + with: + node-version: "24" + cache: pnpm + cache-dependency-path: crates/bashkit-js/pnpm-lock.yaml + - run: pnpm install --frozen-lockfile + working-directory: crates/bashkit-js + - run: pnpm run build + working-directory: crates/bashkit-js + - run: pnpm exec ava __test__/runtime-functions.spec.ts + working-directory: crates/bashkit-js + - run: node --test __test__/runtime-compat/runtime-functions.test.mjs + working-directory: crates/bashkit-js + # Gate job for branch protection js-check: name: JS Check if: always() - needs: [typecheck, build-and-test] + needs: [typecheck, build-and-test, callback-platforms] runs-on: ubuntu-latest steps: - name: Verify all jobs passed @@ -260,3 +291,7 @@ jobs: echo "JS build/test failed" exit 1 fi + if [[ "${{ needs.callback-platforms.result }}" != "success" ]]; then + echo "Runtime callback platform tests failed" + exit 1 + fi diff --git a/Cargo.lock b/Cargo.lock index b7fa1cddb..a4fb66fc7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -518,6 +518,7 @@ dependencies = [ "napi", "napi-build", "napi-derive", + "serde", "serde_json", "tokio", ] diff --git a/contracts/capability-parity.json b/contracts/capability-parity.json index 7bbaa155b..f6dd7c416 100644 --- a/contracts/capability-parity.json +++ b/contracts/capability-parity.json @@ -20,12 +20,14 @@ "runtime_python": "Explicit embedded Python opt-in", "runtime_typescript": "Explicit embedded TypeScript opt-in", "runtime_sqlite": "Explicit embedded SQLite opt-in", - "interactive_terminal": "In-process interactive terminal session (keystrokes in, screen text and per-command transcript out; vi, less)" + "interactive_terminal": "In-process interactive terminal session (keystrokes in, screen text and per-command transcript out; vi, less)", + "host_functions_python": "Named host callbacks invoked from embedded Python", + "host_functions_typescript": "Named host callbacks invoked from embedded TypeScript" }, "surfaces": { "rust_bash": { "label": "Rust BashBuilder", - "supported": ["execution_limits", "cancellation", "deadlines", "output_caps", "cwd", "env", "runtime_env", "stdin", "custom_builtins", "host_call_suspension", "network_policy", "transport_hooks", "virtual_filesystem", "mounts", "snapshots", "runtime_python", "runtime_typescript", "runtime_sqlite", "interactive_terminal"], + "supported": ["execution_limits", "cancellation", "deadlines", "output_caps", "cwd", "env", "runtime_env", "stdin", "custom_builtins", "host_call_suspension", "network_policy", "transport_hooks", "virtual_filesystem", "mounts", "snapshots", "runtime_python", "runtime_typescript", "runtime_sqlite", "interactive_terminal", "host_functions_python", "host_functions_typescript"], "evidence": { "execution_limits": ["crates/bashkit/tests/integration/blackbox_security_tests.rs#max_commands_resets_per_exec"], "cancellation": ["crates/bashkit/tests/integration/cancellation_tests.rs#cancel_aborts_execution"], @@ -45,7 +47,9 @@ "runtime_python": ["crates/bashkit/tests/integration/python_integration_tests.rs#print_hello"], "runtime_typescript": ["crates/bashkit/tests/integration/typescript_integration_tests.rs#bash_writes_file_ts_reads"], "runtime_sqlite": ["crates/bashkit/tests/integration/sqlite_integration_tests.rs#inline_select_returns_value"], - "interactive_terminal": ["crates/bashkit/tests/integration/terminal_tests.rs#agent_edits_config_with_vi_and_reads_result"] + "interactive_terminal": ["crates/bashkit/tests/integration/terminal_tests.rs#agent_edits_config_with_vi_and_reads_result"], + "host_functions_python": ["crates/bashkit/tests/integration/core_extension_point_tests.rs#python_host_function_sees_request_extensions"], + "host_functions_typescript": ["crates/bashkit/tests/integration/core_extension_point_tests.rs#typescript_host_function_sees_request_extensions"] }, "unsupported": { "tool_callbacks": "Use ScriptedTool for named tool callbacks; BashBuilder exposes lower-level custom builtins." @@ -53,7 +57,7 @@ }, "rust_bash_tool": { "label": "Rust BashTool", - "supported": ["execution_limits", "deadlines", "output_caps", "cwd", "env", "custom_builtins", "network_policy", "transport_hooks", "virtual_filesystem", "mounts", "runtime_python", "runtime_typescript", "runtime_sqlite"], + "supported": ["execution_limits", "deadlines", "output_caps", "cwd", "env", "custom_builtins", "network_policy", "transport_hooks", "virtual_filesystem", "mounts", "runtime_python", "runtime_typescript", "runtime_sqlite", "host_functions_python", "host_functions_typescript"], "evidence": { "execution_limits": ["crates/bashkit/src/tool.rs#test_execution_forwards_limits_and_output_caps"], "deadlines": ["crates/bashkit/src/tool.rs#test_tool_execute_timeout"], @@ -67,7 +71,9 @@ "mounts": ["crates/bashkit/src/tool.rs#test_configure_forwards_lazy_mount"], "runtime_python": ["crates/bashkit/src/tool.rs#test_python_runtime_opt_in_executes_through_tool"], "runtime_typescript": ["crates/bashkit/src/tool.rs#test_typescript_runtime_opt_in_executes_through_tool"], - "runtime_sqlite": ["crates/bashkit/src/tool.rs#test_configure_forwards_sqlite_runtime_opt_in"] + "runtime_sqlite": ["crates/bashkit/src/tool.rs#test_configure_forwards_sqlite_runtime_opt_in"], + "host_functions_python": ["crates/bashkit/src/tool.rs#test_python_host_functions_execute_through_tool"], + "host_functions_typescript": ["crates/bashkit/src/tool.rs#test_typescript_host_functions_execute_through_tool"] }, "unsupported": { "runtime_env": "BashTool builds a fresh Bash per execution, so there is no live instance to mutate; env is configured on the builder.", @@ -104,7 +110,9 @@ "runtime_python": "Language runtimes are intentionally absent from the logic-only shell.", "runtime_typescript": "Language runtimes are intentionally absent from the logic-only shell.", "runtime_sqlite": "Language runtimes are intentionally absent from the logic-only shell.", - "interactive_terminal": "ScriptedTool runs logic-only scripts per call and has no terminal device." + "interactive_terminal": "ScriptedTool runs logic-only scripts per call and has no terminal device.", + "host_functions_python": "Language runtimes are intentionally absent from the logic-only shell.", + "host_functions_typescript": "Language runtimes are intentionally absent from the logic-only shell." } }, "cli": { @@ -132,12 +140,14 @@ "transport_hooks": "The CLI exposes allow-all/default-deny policy, not a host transport callback.", "snapshots": "The CLI exposes no snapshot commands.", "runtime_typescript": "The CLI build does not register the TypeScript runtime.", - "interactive_terminal": "The CLI's interactive mode runs on the host's real terminal (rustyline), not the in-process Terminal." + "interactive_terminal": "The CLI's interactive mode runs on the host's real terminal (rustyline), not the in-process Terminal.", + "host_functions_python": "A standalone process cannot receive in-process host callbacks.", + "host_functions_typescript": "A standalone process cannot receive in-process host callbacks." } }, "python": { "label": "Python", - "supported": ["execution_limits", "cancellation", "deadlines", "output_caps", "cwd", "env", "runtime_env", "custom_builtins", "tool_callbacks", "network_policy", "virtual_filesystem", "mounts", "snapshots", "runtime_python", "runtime_typescript", "runtime_sqlite", "interactive_terminal"], + "supported": ["execution_limits", "cancellation", "deadlines", "output_caps", "cwd", "env", "runtime_env", "custom_builtins", "tool_callbacks", "network_policy", "virtual_filesystem", "mounts", "snapshots", "runtime_python", "runtime_typescript", "runtime_sqlite", "interactive_terminal", "host_functions_python", "host_functions_typescript"], "evidence": { "execution_limits": ["crates/bashkit-python/tests/_bashkit_categories.py#test_max_commands_limits_execution"], "cancellation": ["crates/bashkit-python/tests/_bashkit_categories.py#test_bash_cancel_after_reset_still_works"], @@ -155,7 +165,9 @@ "runtime_python": ["crates/bashkit-python/tests/_bashkit_categories.py#test_bash_python_enabled"], "runtime_typescript": ["crates/bashkit-python/tests/test_tool_runtimes.py#test_runtimes_share_virtual_filesystem"], "runtime_sqlite": ["crates/bashkit-python/tests/test_sqlite.py#test_sqlite"], - "interactive_terminal": ["crates/bashkit-python/tests/test_terminal.py#test_call_edits_file_in_vi"] + "interactive_terminal": ["crates/bashkit-python/tests/test_terminal.py#test_call_edits_file_in_vi"], + "host_functions_python": ["crates/bashkit-python/tests/test_external_functions.py#test_async_external_functions_use_caller_loop_and_context"], + "host_functions_typescript": ["crates/bashkit-python/tests/test_external_functions.py#test_async_external_functions_use_caller_loop_and_context"] }, "unsupported": { "stdin": "execute() accepts a command string and optional output callback, not top-level host stdin.", @@ -165,7 +177,7 @@ }, "napi": { "label": "NAPI JavaScript", - "supported": ["execution_limits", "cancellation", "deadlines", "output_caps", "cwd", "env", "runtime_env", "custom_builtins", "tool_callbacks", "network_policy", "virtual_filesystem", "mounts", "snapshots", "runtime_python", "runtime_sqlite", "interactive_terminal"], + "supported": ["execution_limits", "cancellation", "deadlines", "output_caps", "cwd", "env", "runtime_env", "custom_builtins", "tool_callbacks", "network_policy", "virtual_filesystem", "mounts", "snapshots", "runtime_python", "runtime_sqlite", "interactive_terminal", "host_functions_python", "host_functions_typescript", "runtime_typescript"], "evidence": { "execution_limits": ["crates/bashkit-js/__test__/security.spec.ts#WB: command limit is enforced"], "cancellation": ["crates/bashkit-js/__test__/basic.spec.ts#Bash: cancel works after reset"], @@ -182,13 +194,15 @@ "snapshots": ["crates/bashkit-js/__test__/snapshot-history.spec.ts#commit and checkout round-trip shell state and files"], "runtime_python": ["crates/bashkit-js/__test__/integration.spec.ts#integration: python opt-in executes embedded Python"], "runtime_sqlite": ["crates/bashkit-js/__test__/runtime-compat/sqlite.test.mjs#sqlite"], - "interactive_terminal": ["crates/bashkit-js/__test__/terminal.spec.ts#terminal: call edits a file in vi"] + "interactive_terminal": ["crates/bashkit-js/__test__/terminal.spec.ts#terminal: call edits a file in vi"], + "host_functions_python": ["crates/bashkit-js/__test__/runtime-functions.spec.ts#host functions preserve HTTP request contexts and async I/O"], + "host_functions_typescript": ["crates/bashkit-js/__test__/runtime-functions.spec.ts#host functions preserve HTTP request contexts and async I/O"], + "runtime_typescript": ["crates/bashkit-js/__test__/runtime-functions.spec.ts#both runtimes dispatch independently and advertise host functions"] }, "unsupported": { "stdin": "execute() accepts a command string and output callback, not top-level host stdin.", "host_call_suspension": "NAPI custom builtins await callbacks directly; no request and resume handle is exposed.", - "transport_hooks": "NAPI exposes network policy and credentials but no JavaScript HttpTransport hook.", - "runtime_typescript": "The NAPI package does not compile or expose the TypeScript runtime." + "transport_hooks": "NAPI exposes network policy and credentials but no JavaScript HttpTransport hook." } }, "browser_wasm": { @@ -216,7 +230,9 @@ "runtime_python": "The slim browser build omits embedded language runtimes.", "runtime_typescript": "The slim browser build omits embedded language runtimes.", "runtime_sqlite": "The slim browser build omits embedded language runtimes.", - "interactive_terminal": "Not exposed in the slim browser package yet." + "interactive_terminal": "Not exposed in the slim browser package yet.", + "host_functions_python": "The slim browser build currently omits embedded language runtimes.", + "host_functions_typescript": "The slim browser build currently omits embedded language runtimes." } }, "c_abi": { @@ -244,7 +260,9 @@ "runtime_python": "ABI v1 omits embedded language runtimes.", "runtime_typescript": "ABI v1 omits embedded language runtimes.", "runtime_sqlite": "ABI v1 omits embedded language runtimes.", - "interactive_terminal": "Not exposed in the C ABI." + "interactive_terminal": "Not exposed in the C ABI.", + "host_functions_python": "ABI v1 currently omits embedded language runtimes and runtime callback registration.", + "host_functions_typescript": "ABI v1 currently omits embedded language runtimes and runtime callback registration." } } } diff --git a/crates/bashkit-js/.gitignore b/crates/bashkit-js/.gitignore index 818508515..3f9f2d7fe 100644 --- a/crates/bashkit-js/.gitignore +++ b/crates/bashkit-js/.gitignore @@ -55,3 +55,9 @@ Cargo.lock # IDE .vscode-test + +# Runtime host-function bridge output +runtime-functions.js +runtime-functions.d.ts +runtime-functions.js.map +runtime-functions.d.ts.map diff --git a/crates/bashkit-js/Cargo.toml b/crates/bashkit-js/Cargo.toml index e345c4e68..a5d1aad08 100644 --- a/crates/bashkit-js/Cargo.toml +++ b/crates/bashkit-js/Cargo.toml @@ -14,11 +14,12 @@ repository.workspace = true crate-type = ["cdylib"] [dependencies] -bashkit = { path = "../bashkit", features = ["python", "realfs", "jq", "interop", "http_client", "sqlite", "terminal"] } +bashkit = { path = "../bashkit", features = ["python", "typescript", "realfs", "jq", "interop", "http_client", "sqlite", "terminal"] } bashkit-scripted-tool = { path = "../bashkit-scripted-tool", features = ["python", "jq"] } napi = { workspace = true } napi-derive = { workspace = true } serde_json = { workspace = true } +serde = { workspace = true } tokio = { version = "1", features = ["sync", "macros", "io-util", "rt", "time"] } [build-dependencies] diff --git a/crates/bashkit-js/README.md b/crates/bashkit-js/README.md index 5a0f7fc6d..ae6048708 100644 --- a/crates/bashkit-js/README.md +++ b/crates/bashkit-js/README.md @@ -630,10 +630,17 @@ import { - `files?: Record string) | (() => Promise)>` - `mounts?: Array<{ path: string; root: string; writable?: boolean }>` - `python?: boolean` -- `externalFunctions?: string[]` +- `externalFunctions?: string[]`, Python host function names +- `externalHandler?: ExternalHandler`, sync/async Python dispatcher `(name, args, kwargs, { signal })` +- `typescript?: boolean`, opt into embedded TypeScript (`ts`/`node`/`deno`/`bun`) +- `typescriptExternalFunctions?: string[]` +- `typescriptExternalHandler?: TypeScriptExternalHandler`, dispatcher `(name, args, { signal })` + - `customBuiltins?: Record string | Promise>`, JS callbacks registered as bash builtins (see [Custom Builtins](#custom-builtins)) - `network?: NetworkOptions`, outbound HTTP configuration (see [Network](#network)) +Host calls require asynchronous `execute()`. See the [host function guide](../../docs/start-node.md#python-and-typescript-host-functions) for JSON values, caller context, cancellation and reentry rules. + ### ScriptAnalysis Returned by `analyze(script)`. diff --git a/crates/bashkit-js/__test__/fixtures/runtime-functions-worker.mjs b/crates/bashkit-js/__test__/fixtures/runtime-functions-worker.mjs new file mode 100644 index 000000000..bf1da6996 --- /dev/null +++ b/crates/bashkit-js/__test__/fixtures/runtime-functions-worker.mjs @@ -0,0 +1,32 @@ +// A native callback must belong to this worker's environment, even when the +// process-wide Tokio runtime is shared with other workers and they exit first. +import assert from "node:assert/strict"; +import { workerData } from "node:worker_threads"; +import { AsyncLocalStorage } from "node:async_hooks"; +import { BashTool } from "../../wrapper.js"; + +const context = new AsyncLocalStorage(); +const python = workerData === "python"; +const handler = async () => { + await new Promise((resolve) => setTimeout(resolve, 1)); + assert.equal(context.getStore(), "worker"); + return 42; +}; +const bash = new BashTool( + python + ? { python: true, externalFunctions: ["lookup"], externalHandler: handler } + : { + typescript: true, + typescriptExternalFunctions: ["lookup"], + typescriptExternalHandler: handler, + }, +); +for (let i = 0; i < 10; i++) { + const result = await context.run("worker", () => + bash.execute( + python ? "python -c 'print(lookup())'" : "ts -c 'await lookup()'", + ), + ); + assert.equal(result.stdout, "42\n"); + assert.equal(result.exitCode, 0); +} diff --git a/crates/bashkit-js/__test__/runtime-compat/runtime-functions.test.mjs b/crates/bashkit-js/__test__/runtime-compat/runtime-functions.test.mjs new file mode 100644 index 000000000..1f64b461b --- /dev/null +++ b/crates/bashkit-js/__test__/runtime-compat/runtime-functions.test.mjs @@ -0,0 +1,49 @@ +// This suite runs on Node, Bun and Deno against the real native binding. +import test from "node:test"; +import assert from "node:assert/strict"; +import { AsyncLocalStorage } from "node:async_hooks"; +import { Bash, BashTool } from "../../wrapper.js"; + +for (const Factory of [Bash, BashTool]) { + for (const language of ["python", "typescript"]) { + test(`${Factory.name}: ${language} host functions retain async request context`, async () => { + const context = new AsyncLocalStorage(); + const handler = async () => { + await new Promise((resolve) => setTimeout(resolve, 1)); + return context.getStore(); + }; + const bash = new Factory( + language === "python" + ? { + python: true, + externalFunctions: ["lookup"], + externalHandler: handler, + } + : { + typescript: true, + typescriptExternalFunctions: ["lookup"], + typescriptExternalHandler: handler, + }, + ); + const command = + language === "python" + ? "python -c 'print(lookup())'" + : "ts -c 'await lookup()'"; + const results = await Promise.all( + ["first", "second"].map((id) => + context.run(id, () => bash.execute(command)), + ), + ); + assert.deepEqual( + results.map((r) => r.stdout), + ["first\n", "second\n"], + ); + assert.notEqual(bash.executeSync(command).exitCode, 0); + bash.reset(); + assert.equal( + (await context.run("reset", () => bash.execute(command))).stdout, + "reset\n", + ); + }); + } +} diff --git a/crates/bashkit-js/__test__/runtime-functions.spec.ts b/crates/bashkit-js/__test__/runtime-functions.spec.ts new file mode 100644 index 000000000..047567b8d --- /dev/null +++ b/crates/bashkit-js/__test__/runtime-functions.spec.ts @@ -0,0 +1,512 @@ +// Exercise real interpreters and caller contexts; option-forwarding mocks miss +// deadlocks, request-context leakage and callback cancellation. +import test from "ava"; +import { AsyncLocalStorage } from "node:async_hooks"; +import { Worker } from "node:worker_threads"; +import { createServer } from "node:http"; +import { Bash, BashTool } from "../wrapper.js"; +import { createBashTool } from "../langchain.js"; +import { bashTool as aiBashTool } from "../ai.js"; +import { bashTool as anthropicBashTool } from "../anthropic.js"; +import { bashTool as openaiBashTool } from "../openai.js"; + +const request = new AsyncLocalStorage(); +const key = new Uint8Array(32).fill(7); +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +for (const language of ["python", "typescript"] as const) { + const command = + language === "python" + ? `python -c 'print(lookup(21, multiplier=2)["value"])'` + : `ts -c 'const r = await lookup(21, {multiplier: 2}); r.value'`; + const call = + language === "python" + ? "python -c 'print(lookup())'" + : "ts -c 'await lookup()'"; + const options = (handler: (...args: any[]) => any): any => + language === "python" + ? { + python: true, + externalFunctions: ["lookup"], + externalHandler: handler, + } + : { + typescript: true, + typescriptExternalFunctions: ["lookup"], + typescriptExternalHandler: (name: string, args: any[], ctx: any) => + handler(name, args.slice(0, 1), args[1] ?? {}, ctx), + }; + + for (const Factory of [Bash, BashTool]) { + test(`${Factory.name}: ${language} host functions`, async (t) => { + const seen: unknown[] = []; + const bash = new Factory( + options(async (name, args, kwargs) => { + seen.push([name, args, kwargs]); + await delay(1); + return { value: args[0] * kwargs.multiplier }; + }), + ); + const result = await bash.execute(command); + t.is(result.exitCode, 0, result.stderr); + t.is(result.stdout, "42\n"); + t.deepEqual(seen, [["lookup", [21], { multiplier: 2 }]]); + }); + + test(`${Factory.name}: ${language} callbacks retain queued request contexts`, async (t) => { + const bash = new Factory( + options(async () => { + const before = request.getStore(); + await delay(2); + t.is(request.getStore(), before); + return before; + }), + ); + const results = await Promise.all( + Array.from({ length: 8 }, (_, i) => + request.run(String(i), () => bash.execute(call)), + ), + ); + t.deepEqual( + results.map((r) => r.stdout.trim()), + Array.from({ length: 8 }, (_, i) => String(i)), + ); + }); + + test(`${Factory.name}: ${language} reset and snapshots retain supplied handlers`, async (t) => { + const config = options(() => ({ value: 42 })); + const bash = new Factory(config); + bash.writeFile("/tmp/persisted", "data"); + const snapshot = bash.snapshot({ hmacKey: key }); + const restored = + Factory === Bash + ? Bash.fromSnapshot(snapshot, { hmacKey: key }, config) + : BashTool.fromSnapshot(snapshot, config, { hmacKey: key }); + t.is(restored.readFile("/tmp/persisted"), "data"); + for (let i = 0; i < 2; i++) { + t.is((await restored.execute(command)).stdout, "42\n"); + restored.reset(); + } + }); + + test(`${Factory.name}: ${language} timeout aborts host work and permits reuse`, async (t) => { + let aborted = false; + const bash = new Factory({ + ...options(async (_name, _args, _kwargs, ctx) => { + await new Promise((_resolve, reject) => + ctx.signal.addEventListener( + "abort", + () => { + aborted = true; + reject(new Error("aborted")); + }, + { once: true }, + ), + ); + }), + timeoutMs: 2_000, + }); + t.not((await bash.execute(call)).exitCode, 0); + await delay(0); + t.true(aborted); + t.is((await bash.execute("echo reusable")).stdout, "reusable\n"); + }); + + test(`${Factory.name}: ${language} cancellation is scoped to the active request`, async (t) => { + let started!: () => void; + const ready = new Promise((resolve) => { + started = resolve; + }); + let signal: AbortSignal | undefined; + const bash = new Factory( + options(async (_name, _args, _kwargs, ctx) => { + signal = ctx.signal; + started(); + await new Promise((_resolve, reject) => + ctx.signal.addEventListener( + "abort", + () => reject(new Error("cancelled")), + { once: true }, + ), + ); + }), + ); + const controller = new AbortController(); + const active = bash.execute(call, { signal: controller.signal }); + await ready; + const queued = bash.execute("echo next"); + controller.abort(); + t.not((await active).exitCode, 0); + t.true(signal!.aborted); + t.is((await queued).stdout, "next\n"); + }); + + test(`${Factory.name}: ${language} recursive access fails without deadlocking`, async (t) => { + let bash: Bash | BashTool; + bash = new Factory( + options(async () => { + await bash.execute("echo recursive"); + }), + ); + const result = await bash.execute(call); + t.not(result.exitCode, 0); + t.regex(result.stderr + (result.error ?? ""), /re-enter/); + t.is((await bash.execute("echo reusable")).stdout, "reusable\n"); + }); + + test(`${Factory.name}: ${language} retained live filesystem access fails promptly`, async (t) => { + let fs: ReturnType; + const bash = new Factory(options(() => fs.readFile("/tmp/state"))); + bash.writeFile("/tmp/state", "data"); + fs = bash.fs(); + const result = await bash.execute(call); + t.not(result.exitCode, 0); + t.regex(result.stderr + (result.error ?? ""), /re-enter|would block/); + t.is(fs.readFile("/tmp/state"), "data"); + }); + + test(`${Factory.name}: ${language} sync access racing a new handler cannot deadlock`, async (t) => { + const bash = new Factory(options(() => 42)); + bash.writeFile("/tmp/state", "ready"); + let started!: () => void; + const ready = new Promise((resolve) => { + started = resolve; + }); + const executing = bash.execute(`echo started; sleep 0.2; ${call}`, { + onOutput: () => { + started(); + }, + }); + await ready; + await delay(0); + t.throws(() => bash.readFile("/tmp/state"), { message: /would block/ }); + const result = await executing; + t.is(result.exitCode, 0, result.stderr); + t.is(result.stdout, "started\n42\n"); + }); + + test(`${Factory.name}: ${language} sync invocation rejects host calls`, (t) => { + let called = false; + const bash = new Factory( + options(() => { + called = true; + return 42; + }), + ); + const result = bash.executeSync(call); + t.not(result.exitCode, 0); + t.regex(result.stderr + (result.error ?? ""), /execute\(\).*async/); + t.false(called); + t.is(bash.executeSync("echo normal").stdout, "normal\n"); + }); + } + + for (const Factory of [Bash, BashTool]) { + test(`${Factory.name}: ${language} unrelated sync calls fail while async calls queue`, async (t) => { + let started!: () => void, release!: () => void; + const ready = new Promise((resolve) => { + started = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); + const bash = new Factory( + options(async () => { + started(); + await gate; + return 42; + }), + ); + bash.writeFile("/tmp/state", "data"); + const fs = bash.fs(); + const active = bash.execute(call); + await ready; + for (const operation of [ + () => bash.executeSync("echo unsafe"), + () => bash.readFile("/tmp/state"), + () => fs.readFile("/tmp/state"), + () => bash.snapshot({ hmacKey: key }), + () => bash.reset(), + ]) { + t.throws(operation, { message: /would block/ }); + } + const queued = bash.execute("echo queued"); + release(); + t.is((await active).stdout, "42\n"); + t.is((await queued).stdout, "queued\n"); + }); + + test(`${Factory.name}: ${language} queued cancellation preserves the active callback`, async (t) => { + let started!: () => void, release!: () => void; + let activeSignal!: AbortSignal; + const ready = new Promise((resolve) => { + started = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); + const bash = new Factory( + options(async (_name, _args, _kwargs, ctx) => { + activeSignal = ctx.signal; + started(); + await gate; + return 42; + }), + ); + const active = bash.execute(call); + await ready; + const controller = new AbortController(); + const queued = bash.execute("echo forbidden > /tmp/cancelled", { + signal: controller.signal, + }); + controller.abort(); + t.false(activeSignal.aborted); + release(); + t.is((await active).stdout, "42\n"); + t.not((await queued).exitCode, 0); + t.false(bash.exists("/tmp/cancelled")); + const restored = Factory.fromSnapshotKeyed( + bash.snapshotKeyed(key), + key, + options(() => 21), + ); + t.is((await restored.execute(call)).stdout, "21\n"); + }); + } + + test(`${language} host functions preserve HTTP request contexts and async I/O`, async (t) => { + const upstream = createServer((req, res) => res.end(req.url!.slice(1))); + await new Promise((resolve) => + upstream.listen(0, "127.0.0.1", resolve), + ); + const upstreamPort = (upstream.address() as { port: number }).port; + const bash = new BashTool( + options(async (_name, _args, _kwargs, ctx) => { + const id = request.getStore(); + const response = await fetch(`http://127.0.0.1:${upstreamPort}/${id}`, { + signal: ctx.signal, + }); + t.is(request.getStore(), id); + return response.text(); + }), + ); + const server = createServer((req, res) => { + void request.run(req.url!.slice(1), async () => { + try { + res.end((await bash.execute(call)).stdout.trim()); + } catch (error) { + res.writeHead(500).end(String(error)); + } + }); + }); + await new Promise((resolve) => + server.listen(0, "127.0.0.1", resolve), + ); + const port = (server.address() as { port: number }).port; + try { + const responses = await Promise.all( + Array.from({ length: 8 }, (_, i) => + fetch(`http://127.0.0.1:${port}/${i}`).then((r) => r.text()), + ), + ); + t.deepEqual( + responses, + Array.from({ length: 8 }, (_, i) => String(i)), + ); + } finally { + await Promise.all( + [server, upstream].map( + (service) => + new Promise((resolve) => service.close(() => resolve())), + ), + ); + } + }); + + test(`${language} callbacks remain safe across worker environments and exit`, async (t) => { + const workers = Array.from( + { length: 6 }, + () => + new Worker( + new URL("./fixtures/runtime-functions-worker.mjs", import.meta.url), + { workerData: language }, + ), + ); + await Promise.all( + workers.map( + (worker) => + new Promise((resolve, reject) => { + worker.on("error", reject); + worker.on("exit", (code) => + code === 0 + ? resolve() + : reject(new Error(`worker exited ${code}`)), + ); + }), + ), + ); + t.pass(); + }); + + test(`${language} rejects unsupported guest arguments before invoking the host`, async (t) => { + let invoked = false; + const bash = new Bash( + options(() => { + invoked = true; + return 42; + }), + ); + for (const script of language === "python" + ? ["lookup(b'bytes')", "lookup({1: 2})", "lookup(9007199254740993)"] + : ["await lookup(() => 42)", "await lookup(9007199254740993)"]) { + const result = await bash.execute( + `${language === "python" ? "python" : "ts"} -c ${JSON.stringify(script)}`, + ); + t.not(result.exitCode, 0); + } + t.false(invoked); + }); + + test(`${language} exception messages hide host stacks and paths`, async (t) => { + const bash = new Bash( + options(() => { + throw new Error( + "failed at /home/host/secret.ts and C:\\private\\file.ts", + ); + }), + ); + const result = await bash.execute(call); + t.not(result.exitCode, 0); + t.false((result.stderr + result.error).includes("secret.ts")); + t.false((result.stderr + result.error).includes("file.ts")); + }); + + test(`LangChain forwards ${language} runtime callbacks`, async (t) => { + const tool = createBashTool(options(() => ({ value: 42 }))); + t.is(await tool.invoke({ commands: command }), "42\n"); + }); + + test(`AI adapters execute ${language} runtime callbacks`, async (t) => { + const config = options(() => ({ value: 42 })); + const ai = aiBashTool(config); + t.is((await ai.tools.bash.execute({ commands: command })).trim(), "42"); + const anthropic = anthropicBashTool(config); + t.is( + ( + await anthropic.handler({ + type: "tool_use", + id: "host", + name: "bash", + input: { commands: command }, + }) + ).content.trim(), + "42", + ); + const openai = openaiBashTool(config); + t.is( + ( + await openai.handler({ + id: "host", + type: "function", + function: { + name: "bash", + arguments: JSON.stringify({ commands: command }), + }, + }) + ).content.trim(), + "42", + ); + }); + + test(`LangChain cancellation reaches ${language} host work`, async (t) => { + let ready!: () => void; + const started = new Promise((resolve) => { + ready = resolve; + }); + let signal: AbortSignal | undefined; + const tool = createBashTool( + options(async (_name, _args, _kwargs, ctx) => { + signal = ctx.signal; + ready(); + await new Promise((_resolve, reject) => + ctx.signal.addEventListener( + "abort", + () => reject(new Error("cancelled")), + { once: true }, + ), + ); + }), + ); + const controller = new AbortController(); + const execution = tool.invoke( + { commands: call }, + { signal: controller.signal }, + ); + // LangChain itself may reject first; attach immediately to avoid an unhandled rejection. + const settled = execution.catch(() => undefined); + await started; + controller.abort(); + await settled; + for (let i = 0; i < 100 && !signal?.aborted; i++) await delay(5); + t.true(signal!.aborted); + }); + + test(`${language} callback errors and invalid values remain script failures`, async (t) => { + for (const handler of [ + () => { + throw new Error("host failure"); + }, + () => new Date(), + () => NaN, + () => 2 ** 60, + () => () => 42, + () => { + const value: unknown[] = []; + value.push(value); + return value; + }, + ]) { + const bash = new Bash(options(handler)); + t.not((await bash.execute(call)).exitCode, 0); + t.is((await bash.execute("echo recovered")).stdout, "recovered\n"); + } + }); +} + +test("runtime function configuration is validated", (t) => { + t.throws(() => new Bash({ externalFunctions: ["lookup"] }), { + message: /externalHandler/, + }); + t.throws(() => new Bash({ typescriptExternalFunctions: ["lookup"] } as any), { + message: /typescriptExternalHandler/, + }); + t.throws(() => new Bash({ externalHandler: () => 42 } as any), { + message: /python/, + }); + t.throws(() => new Bash({ typescriptExternalHandler: () => 42 } as any), { + message: /typescript/, + }); +}); + +test("both runtimes dispatch independently and advertise host functions", async (t) => { + const bash = new BashTool({ + python: true, + typescript: true, + externalFunctions: ["lookup"], + externalHandler: () => 21, + typescriptExternalFunctions: ["lookup"], + typescriptExternalHandler: (_name, args) => Number(args[0]) * 2, + }); + const result = await bash.execute( + `python -c 'from pathlib import Path; n = Path("/tmp/value").write_text(str(lookup()))'; ts -c 'await lookup(await readFile("/tmp/value"))'`, + ); + t.is(result.stdout, "42\n"); + for (const metadata of [ + bash.description(), + bash.help(), + bash.systemPrompt(), + ]) { + t.regex(metadata, /Python host functions: lookup/); + t.regex(metadata, /TypeScript host functions: lookup/); + } +}); diff --git a/crates/bashkit-js/langchain.ts b/crates/bashkit-js/langchain.ts index d567df968..8fd34932b 100644 --- a/crates/bashkit-js/langchain.ts +++ b/crates/bashkit-js/langchain.ts @@ -82,13 +82,15 @@ export function createBashTool( return new DynamicStructuredTool({ name: bashTool.name, description: [ - bashTool.shortDescription, + bashTool.description(), "Execute bash commands in a sandboxed virtual filesystem.", "State persists between calls. Use for file operations, text processing, and scripting.", ].join(" "), schema: bashInputSchema, - func: async ({ commands }: { commands: string }) => { - const result = bashTool.executeSync(commands); + func: async ({ commands }: { commands: string }, _runManager, config) => { + const result = await bashTool.execute(commands, { + signal: config?.signal, + }); return formatResult(result); }, }); diff --git a/crates/bashkit-js/package.json b/crates/bashkit-js/package.json index 45eca201c..3e2de177e 100644 --- a/crates/bashkit-js/package.json +++ b/crates/bashkit-js/package.json @@ -57,6 +57,8 @@ "wrapper.d.ts", "scripted.js", "scripted.d.ts", + "runtime-functions.js", + "runtime-functions.d.ts", "internal.js", "internal.d.ts", "langchain.js", diff --git a/crates/bashkit-js/runtime-functions.ts b/crates/bashkit-js/runtime-functions.ts new file mode 100644 index 000000000..736ee66e8 --- /dev/null +++ b/crates/bashkit-js/runtime-functions.ts @@ -0,0 +1,239 @@ +// Host dispatch is execution-local: bind request AsyncLocalStorage before a +// call can queue, and revoke its AbortSignal when native execution ends. +// A task-local marker rejects recursive calls while unrelated requests queue. +import { AsyncLocalStorage, AsyncResource } from "node:async_hooks"; +import type { ExecResult } from "./index.cjs"; + +export type RuntimeValue = + | null + | boolean + | number + | string + | RuntimeValue[] + | { [key: string]: RuntimeValue }; + +export interface HostFunctionContext { + /** Aborted when execution is cancelled, times out, or completes. */ + signal: AbortSignal; +} + +export type ExternalHandler = ( + name: string, + args: RuntimeValue[], + kwargs: Record, + context: HostFunctionContext, +) => RuntimeValue | undefined | Promise; + +export type TypeScriptExternalHandler = ( + name: string, + args: RuntimeValue[], + context: HostFunctionContext, +) => RuntimeValue | undefined | Promise; + +export interface RuntimeFunctionOptions { + python?: boolean; + typescript?: boolean; + externalFunctions?: string[]; + externalHandler?: ExternalHandler; + typescriptExternalFunctions?: string[]; + typescriptExternalHandler?: TypeScriptExternalHandler; +} + +const handlers = new WeakMap(); +const callbackOwners = new AsyncLocalStorage>(); + +export function assertRuntimeAccess(owner: object): void { + if (callbackOwners.getStore()?.has(owner)) { + throw new Error( + "external handler cannot re-enter the same interpreter or its live filesystem", + ); + } +} + +export function validateRuntimeFunctions( + options?: RuntimeFunctionOptions, +): void { + for (const [language, names, handler, label] of [ + [ + "python", + options?.externalFunctions, + options?.externalHandler, + "externalHandler", + ], + [ + "typescript", + options?.typescriptExternalFunctions, + options?.typescriptExternalHandler, + "typescriptExternalHandler", + ], + ] as const) { + if (names?.length && !handler) + throw new TypeError(`${label} is required with external functions`); + if (handler !== undefined) { + if (typeof handler !== "function") + throw new TypeError(`${label} must be callable`); + if (!options?.[language]) + throw new TypeError(`${label} requires ${language}: true`); + } + } +} + +export function configureRuntimeFunctions( + native: T, + options?: RuntimeFunctionOptions, +): T { + validateRuntimeFunctions(options); + if (!options?.externalHandler && !options?.typescriptExternalHandler) + return native; + // Protect all public instance operations, including methods used by adapters. + // Native sync entry points separately protect retained/raw filesystem handles. + const owner = new Proxy(native, { + get(target, key) { + if (key !== "cancel" && key !== "clearCancel") assertRuntimeAccess(owner); + const value = Reflect.get(target, key, target); + return typeof value === "function" + ? (...args: unknown[]) => { + if (key !== "cancel" && key !== "clearCancel") + assertRuntimeAccess(owner); + return Reflect.apply(value, target, args); + } + : value; + }, + }); + handlers.set(owner, { ...options }); + return owner; +} + +// Build a fresh JSON tree: reject non-finite/unsafe numbers, functions, +// arbitrary objects and cyclic/deep values rather than silently coercing them. +function runtimeValue(value: unknown, depth = 0): RuntimeValue { + if (depth > 64) + throw new RangeError("host value nesting depth exceeds maximum of 64"); + if (value == null) return null; + if (typeof value === "string" || typeof value === "boolean") return value; + if (typeof value === "number") { + if (!Number.isFinite(value)) + throw new TypeError("host numbers must be finite"); + if (Number.isInteger(value) && !Number.isSafeInteger(value)) { + throw new RangeError("host integers must be safe JavaScript integers"); + } + return value; + } + if (Array.isArray(value)) + return value.map((item) => runtimeValue(item, depth + 1)); + if ( + typeof value === "object" && + [null, Object.prototype].includes(Object.getPrototypeOf(value)) + ) { + // fromEntries creates own data properties, including the __proto__ key. + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [ + key, + runtimeValue(item, depth + 1), + ]), + ); + } + throw new TypeError("external handlers require JSON values"); +} + +async function abortable( + invoke: () => T | Promise, + signal: AbortSignal, +): Promise { + if (signal.aborted) throw new Error("host execution is no longer active"); + let abort!: () => void; + const cancelled = new Promise((_resolve, reject) => { + abort = () => reject(new Error("host execution is no longer active")); + signal.addEventListener("abort", abort, { once: true }); + }); + try { + return await Promise.race([Promise.resolve().then(invoke), cancelled]); + } finally { + signal.removeEventListener("abort", abort); + } +} + +export interface RuntimeSession { + dispatch: (request: string) => Promise; + finish(): void; +} + +export function captureRuntimeSession( + owner: object, +): RuntimeSession | undefined { + assertRuntimeAccess(owner); + const config = handlers.get(owner); + if (!config) return undefined; + const controller = new AbortController(); + const context = { signal: controller.signal }; + const dispatch = AsyncResource.bind(async (request: string) => { + try { + if (context.signal.aborted) + throw new Error("host execution is no longer active"); + const payload = JSON.parse(request); + const args = runtimeValue(payload.args) as RuntimeValue[]; + const kwargs = runtimeValue(payload.kwargs ?? {}) as Record< + string, + RuntimeValue + >; + const owners = new Set(callbackOwners.getStore()); + owners.add(owner); + const result = await callbackOwners.run(owners, () => + abortable(() => { + if (payload.language === "python" && config.externalHandler) { + return config.externalHandler(payload.name, args, kwargs, context); + } + if ( + payload.language === "typescript" && + config.typescriptExternalHandler + ) { + return config.typescriptExternalHandler( + payload.name, + args, + context, + ); + } + throw new Error("no handler configured for this runtime"); + }, context.signal), + ); + return JSON.stringify({ status: "ok", value: runtimeValue(result) }); + } catch (error) { + // THREAT[TM-INF-028]: expose a bounded message, never a host stack/path. + const message = (error instanceof Error ? error.message : String(error)) + .replace(/file:\/\/[^\s]*/g, "") + .replace(/(?") + .replace(/[A-Za-z]:[\\/][^\s]*/g, "") + .slice(0, 256); + return JSON.stringify({ + status: "error", + value: message || "host callback failed", + }); + } + }); + return { dispatch, finish: () => controller.abort() }; +} + +export function executeNativeRuntime( + native: { + execute(commands: string): Promise; + executeWithOutput(commands: string, output: any): Promise; + }, + commands: string, + output: unknown, + session?: RuntimeSession, +): Promise { + if (session) { + return ( + native as typeof native & { + executeWithRuntime( + commands: string, + dispatch: RuntimeSession["dispatch"], + output?: unknown, + ): Promise; + } + ).executeWithRuntime(commands, session.dispatch, output); + } + return output + ? native.executeWithOutput(commands, output) + : native.execute(commands); +} diff --git a/crates/bashkit-js/src/lib.rs b/crates/bashkit-js/src/lib.rs index 4b935d171..74939949e 100644 --- a/crates/bashkit-js/src/lib.rs +++ b/crates/bashkit-js/src/lib.rs @@ -13,6 +13,7 @@ //! work. This prevents CodeQL `rust/access-invalid-pointer` alerts caused by //! holding a raw-pointer-derived `&self` across `block_on` or `.await` points. +mod runtime_functions; mod terminal; use bashkit::interop::fs::{ @@ -21,10 +22,9 @@ use bashkit::interop::fs::{ use bashkit::tool::VERSION; use bashkit::{ Bash as RustBash, BashTool as RustBashTool, Builtin, BuiltinContext, BuiltinRegistry, - Credential, ExecResult as RustExecResult, ExecutionLimits, ExtFunctionResult, - FileSystem as BashFileSystem, FileType, InMemoryFs, Metadata, MontyObject, NetworkAllowlist, - OutputCallback, PosixFs, PythonExternalFnHandler, ReadOnlyFs, RealFs, RealFsMode, - SnapshotOptions as RustSnapshotOptions, Tool, ToolRequest, async_trait, + Credential, ExecResult as RustExecResult, ExecutionLimits, FileSystem as BashFileSystem, + FileType, InMemoryFs, Metadata, NetworkAllowlist, OutputCallback, PosixFs, ReadOnlyFs, RealFs, + RealFsMode, SnapshotOptions as RustSnapshotOptions, Tool, ToolRequest, async_trait, }; use bashkit::{ CapabilityFingerprint as RustCapabilityFingerprint, CheckoutPolicy as RustCheckoutPolicy, @@ -39,7 +39,6 @@ use std::collections::HashMap; use std::ffi::c_void; use std::mem::{MaybeUninit, size_of}; use std::path::{Path, PathBuf}; -use std::pin::Pin; use std::ptr; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex as StdMutex}; @@ -121,116 +120,23 @@ fn reject_on_output_reentry(state: &Arc) -> napi::Result<()> { struct SyncExecuteScope { depth: Arc, + wake: Option>, } impl SyncExecuteScope { fn enter(depth: Arc) -> Self { depth.fetch_add(1, Ordering::SeqCst); - Self { depth } + Self { depth, wake: None } } } impl Drop for SyncExecuteScope { fn drop(&mut self) { self.depth.fetch_sub(1, Ordering::SeqCst); - } -} - -// ============================================================================ -// MontyObject <-> JSON conversion -// ============================================================================ - -#[allow(dead_code)] -fn monty_to_json(obj: &MontyObject) -> serde_json::Value { - match obj { - MontyObject::None => serde_json::Value::Null, - MontyObject::Bool(b) => serde_json::Value::Bool(*b), - MontyObject::Int(i) => serde_json::json!(*i), - MontyObject::BigInt(b) => serde_json::Value::String(b.to_string()), - MontyObject::Float(f) => serde_json::json!(*f), - MontyObject::String(s) | MontyObject::Path(s) => serde_json::Value::String(s.clone()), - MontyObject::Bytes(b) => serde_json::Value::String(base64_encode(b)), - MontyObject::Tuple(items) | MontyObject::List(items) => { - serde_json::Value::Array(items.iter().map(monty_to_json).collect()) - } - MontyObject::Set(items) | MontyObject::FrozenSet(items) => { - serde_json::Value::Array(items.iter().map(monty_to_json).collect()) - } - MontyObject::Dict(pairs) => { - let mut map = serde_json::Map::new(); - for (k, v) in pairs.clone() { - let key = match &k { - MontyObject::String(s) => s.clone(), - other => format!("{}", monty_to_json(other)), - }; - map.insert(key, monty_to_json(&v)); - } - serde_json::Value::Object(map) - } - MontyObject::NamedTuple { - field_names, - values, - .. - } => { - let mut map = serde_json::Map::new(); - for (name, value) in field_names.iter().zip(values.iter()) { - map.insert(name.clone(), monty_to_json(value)); - } - serde_json::Value::Object(map) - } - other => serde_json::Value::String(other.py_repr()), - } -} - -#[allow(dead_code)] -fn json_to_monty(val: &serde_json::Value) -> MontyObject { - match val { - serde_json::Value::Null => MontyObject::None, - serde_json::Value::Bool(b) => MontyObject::Bool(*b), - serde_json::Value::Number(n) => { - if let Some(i) = n.as_i64() { - MontyObject::Int(i) - } else if let Some(f) = n.as_f64() { - MontyObject::Float(f) - } else { - MontyObject::None - } - } - serde_json::Value::String(s) => MontyObject::String(s.clone()), - serde_json::Value::Array(arr) => MontyObject::List(arr.iter().map(json_to_monty).collect()), - serde_json::Value::Object(map) => { - let pairs: Vec<(MontyObject, MontyObject)> = map - .iter() - .map(|(k, v)| (MontyObject::String(k.clone()), json_to_monty(v))) - .collect(); - MontyObject::dict(pairs) - } - } -} - -#[allow(dead_code)] -fn base64_encode(data: &[u8]) -> String { - const CHARS: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - let mut result = String::with_capacity(data.len().div_ceil(3) * 4); - for chunk in data.chunks(3) { - let b0 = chunk[0] as u32; - let b1 = chunk.get(1).copied().unwrap_or(0) as u32; - let b2 = chunk.get(2).copied().unwrap_or(0) as u32; - let n = (b0 << 16) | (b1 << 8) | b2; - result.push(CHARS[(n >> 18 & 63) as usize] as char); - result.push(CHARS[(n >> 12 & 63) as usize] as char); - if chunk.len() > 1 { - result.push(CHARS[(n >> 6 & 63) as usize] as char); - } else { - result.push('='); - } - if chunk.len() > 2 { - result.push(CHARS[(n & 63) as usize] as char); - } else { - result.push('='); + if let Some(wake) = &self.wake { + wake.notify_waiters(); } } - result } // ============================================================================ @@ -1347,6 +1253,10 @@ pub struct BashOptions { pub python: Option, /// Names of external functions callable from embedded Python code. pub external_functions: Option>, + /// Enable embedded TypeScript execution. + pub typescript: Option, + /// Names of TypeScript host functions. + pub typescript_external_functions: Option>, /// Enable the embedded SQLite builtin (`sqlite`/`sqlite3`). /// /// Backed by Turso. When `true`, the binding both registers the @@ -1511,6 +1421,8 @@ fn default_opts() -> BashOptions { allowed_mount_paths: None, python: None, external_functions: None, + typescript: None, + typescript_external_functions: None, sqlite: None, network: None, } @@ -1555,11 +1467,13 @@ struct SharedState { cancelled: std::sync::Mutex>, on_output_reentry_depth: Arc, /// Tracks whether the JS event loop is currently blocked inside an - /// `executeSync` call. Custom builtins dispatched through a threadsafe + /// blocking instance operation. Custom builtins dispatched through a threadsafe /// function check this and fail closed instead of deadlocking on a /// callback the loop can never service. See [`SYNC_BUILTIN_DEADLOCK_ERROR`]. in_sync_execute_depth: Arc, async_execute_semaphore: Arc, + async_execute_depth: Arc, + sync_idle: Arc, username: Option, profile: Option, hostname: Option, @@ -1584,11 +1498,12 @@ struct SharedState { mounts: Option>, allowed_mount_paths: Option>, python: bool, + typescript: bool, + typescript_external_functions: Vec, sqlite: bool, /// Validated network configuration (see [`validate_network_options`]). network: Option, external_functions: Vec, - external_handler: Option, /// Host-owned mutable registry of custom builtins. The same handle is /// passed to the bashkit builder *and* retained here so JS-side /// `addBuiltin()` calls insert into the live interpreter without rebuilding @@ -1688,17 +1603,6 @@ fn forget_runtime_mount(log: &RuntimeMountLog, vfs_path: &str) { mounts.retain(|existing| Path::new(existing.vfs_path()) != normalized); } -/// Wrapper for the external handler that can be stored and cloned. -type ExternalHandlerArc = Arc< - dyn Fn( - String, - Vec, - Vec<(MontyObject, MontyObject)>, - ) -> Pin + Send>> - + Send - + Sync, ->; - /// Clone `Arc`, then use the runtime to block on a future that /// captures only the cloned Arc. This avoids holding raw `&self` across /// `block_on` boundaries. @@ -1710,6 +1614,15 @@ where Fut: std::future::Future>, { reject_on_output_reentry(state)?; + // Publish blocking entry before checking async work. The async admission + // path waits if it starts after this check, leaving this event loop free. + let mut _sync_scope = SyncExecuteScope::enter(state.in_sync_execute_depth.clone()); + _sync_scope.wake = Some(state.sync_idle.clone()); + if state.async_execute_depth.load(Ordering::SeqCst) > 0 { + return Err(napi::Error::from_reason( + "synchronous access would block the event loop of an active execution; use execute() or another instance", + )); + } let s = state.clone(); let rt_guard = s.rt.blocking_lock(); let s2 = state.clone(); @@ -1734,12 +1647,40 @@ fn validate_async_execute_input(state: &SharedState, commands: &str) -> Option) -> napi::Result { - state +struct AsyncExecuteScope { + _permit: OwnedSemaphorePermit, + depth: Arc, +} +impl Drop for AsyncExecuteScope { + fn drop(&mut self) { + self.depth.fetch_sub(1, Ordering::SeqCst); + } +} + +async fn acquire_async_execute_slot(state: &Arc) -> napi::Result { + let permit = state .async_execute_semaphore .clone() .try_acquire_owned() - .map_err(|_| napi::Error::from_reason(ASYNC_EXECUTE_QUEUE_FULL_ERROR)) + .map_err(|_| napi::Error::from_reason(ASYNC_EXECUTE_QUEUE_FULL_ERROR))?; + state.async_execute_depth.fetch_add(1, Ordering::SeqCst); + let scope = AsyncExecuteScope { + _permit: permit, + depth: state.async_execute_depth.clone(), + }; + // Dual SeqCst counters ensure at least one contender observes the other. + // Async entry waits for an earlier blocking entry; blocking entry rejects + // an earlier async execution, whose output or host callback needs JS. + loop { + let notified = state.sync_idle.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if state.in_sync_execute_depth.load(Ordering::SeqCst) == 0 { + break; + } + notified.await; + } + Ok(scope) } fn canonicalize_path(path: &str, label: &str) -> napi::Result { @@ -1800,7 +1741,7 @@ impl Bash { #[napi(constructor)] pub fn new(options: Option) -> napi::Result { let opts = options.unwrap_or_else(default_opts); - let state = shared_state_from_opts(opts, None)?; + let state = shared_state_from_opts(opts)?; Ok(Self { state: Arc::new(state), }) @@ -1815,7 +1756,6 @@ impl Bash { commands: String, on_output: Option>>, ) -> napi::Result { - let _sync_scope = SyncExecuteScope::enter(self.state.in_sync_execute_depth.clone()); let env_raw = on_output .as_ref() .map(|on_output| on_output.value().env as usize); @@ -1851,6 +1791,24 @@ impl Bash { }) } + /// Execute with an execution-local host-function dispatcher. + #[napi( + js_name = "executeWithRuntime", + ts_args_type = "commands: string, dispatcher: (request: string) => Promise, onOutput?: (chunkPair: [string, string]) => string | undefined" + )] + pub fn execute_with_runtime<'env>( + &self, + commands: String, + dispatcher: napi::bindgen_prelude::Function< + 'env, + String, + napi::bindgen_prelude::Promise, + >, + on_output: Option>>, + ) -> napi::Result> { + runtime_functions::execute(self.state.clone(), commands, dispatcher, on_output) + } + /// Execute bash commands asynchronously, returning a Promise. #[napi] pub async fn execute(&self, commands: String) -> napi::Result { @@ -1859,7 +1817,7 @@ impl Bash { return Ok(result); } let s = self.state.clone(); - let _slot = acquire_async_execute_slot(&s)?; + let _slot = acquire_async_execute_slot(&s).await?; let mut bash = s.inner.lock().await; execute_rust_bash(&mut bash, &commands, None, None, None, None).await } @@ -1884,7 +1842,7 @@ impl Bash { if let Some(result) = validate_async_execute_input(&state, &commands) { return Ok(result); } - let _slot = acquire_async_execute_slot(&state)?; + let _slot = acquire_async_execute_slot(&state).await?; let mut bash = state.inner.lock().await; let cancelled = bash.cancellation_token(); let callback_requested_cancel = Arc::new(AtomicBool::new(false)); @@ -2217,7 +2175,7 @@ impl Bash { snapshot_options: Option, ) -> napi::Result { let opts = options.unwrap_or_else(default_opts); - let mut state = shared_state_from_opts(opts, None)?; + let mut state = shared_state_from_opts(opts)?; // restore_snapshot preserves the instance's limits while restoring shell state if let Some(key) = snapshot_hmac_key(snapshot_options.as_ref()) { @@ -2242,7 +2200,7 @@ impl Bash { options: Option, ) -> napi::Result { let opts = options.unwrap_or_else(default_opts); - let mut state = shared_state_from_opts(opts, None)?; + let mut state = shared_state_from_opts(opts)?; state .inner @@ -2575,6 +2533,21 @@ pub struct BashTool { } impl BashTool { + fn with_runtime_function_help(&self, mut text: String) -> String { + for (language, names) in [ + ("Python", &self.state.external_functions), + ("TypeScript", &self.state.typescript_external_functions), + ] { + if !names.is_empty() { + text.push_str(&format!( + "\n\n{language} host functions: {}. Use the host-provided signatures.", + names.join(", ") + )); + } + } + text + } + fn build_rust_tool(state: &SharedState) -> RustBashTool { let mut builder = RustBashTool::builder(); @@ -2593,6 +2566,12 @@ impl BashTool { } } + if state.python { + builder = builder.python(); + } + if state.typescript { + builder = builder.typescript(); + } builder.limits(build_limits(state)).build() } } @@ -2602,7 +2581,7 @@ impl BashTool { #[napi(constructor)] pub fn new(options: Option) -> napi::Result { let opts = options.unwrap_or_else(default_opts); - let state = shared_state_from_opts(opts, None)?; + let state = shared_state_from_opts(opts)?; Ok(Self { state: Arc::new(state), }) @@ -2617,7 +2596,6 @@ impl BashTool { commands: String, on_output: Option>>, ) -> napi::Result { - let _sync_scope = SyncExecuteScope::enter(self.state.in_sync_execute_depth.clone()); let env_raw = on_output .as_ref() .map(|on_output| on_output.value().env as usize); @@ -2653,6 +2631,24 @@ impl BashTool { }) } + /// Execute with an execution-local host-function dispatcher. + #[napi( + js_name = "executeWithRuntime", + ts_args_type = "commands: string, dispatcher: (request: string) => Promise, onOutput?: (chunkPair: [string, string]) => string | undefined" + )] + pub fn execute_with_runtime<'env>( + &self, + commands: String, + dispatcher: napi::bindgen_prelude::Function< + 'env, + String, + napi::bindgen_prelude::Promise, + >, + on_output: Option>>, + ) -> napi::Result> { + runtime_functions::execute(self.state.clone(), commands, dispatcher, on_output) + } + /// Execute bash commands asynchronously, returning a Promise. #[napi] pub async fn execute(&self, commands: String) -> napi::Result { @@ -2661,7 +2657,7 @@ impl BashTool { return Ok(result); } let s = self.state.clone(); - let _slot = acquire_async_execute_slot(&s)?; + let _slot = acquire_async_execute_slot(&s).await?; let mut bash = s.inner.lock().await; execute_rust_bash(&mut bash, &commands, None, None, None, None).await } @@ -2686,7 +2682,7 @@ impl BashTool { if let Some(result) = validate_async_execute_input(&state, &commands) { return Ok(result); } - let _slot = acquire_async_execute_slot(&state)?; + let _slot = acquire_async_execute_slot(&state).await?; let mut bash = state.inner.lock().await; let cancelled = bash.cancellation_token(); let callback_requested_cancel = Arc::new(AtomicBool::new(false)); @@ -2879,7 +2875,7 @@ impl BashTool { ) -> napi::Result { let opts = options.unwrap_or_else(default_opts); let key = require_snapshot_hmac_key(snapshot_options.as_ref())?.to_vec(); - let mut state = shared_state_from_opts(opts, None)?; + let mut state = shared_state_from_opts(opts)?; state .inner @@ -2903,7 +2899,7 @@ impl BashTool { options: Option, ) -> napi::Result { let opts = options.unwrap_or_else(default_opts); - let mut state = shared_state_from_opts(opts, None)?; + let mut state = shared_state_from_opts(opts)?; state .inner @@ -2931,19 +2927,21 @@ impl BashTool { /// Get token-efficient tool description. #[napi] pub fn description(&self) -> String { - Self::build_rust_tool(&self.state).description().to_string() + self.with_runtime_function_help( + Self::build_rust_tool(&self.state).description().to_string(), + ) } /// Get help as a Markdown document. #[napi] pub fn help(&self) -> String { - Self::build_rust_tool(&self.state).help() + self.with_runtime_function_help(Self::build_rust_tool(&self.state).help()) } /// Get compact system-prompt text for orchestration. #[napi] pub fn system_prompt(&self) -> String { - Self::build_rust_tool(&self.state).system_prompt() + self.with_runtime_function_help(Self::build_rust_tool(&self.state).system_prompt()) } /// Get JSON input schema as string. @@ -3814,23 +3812,28 @@ fn build_bash_from_state(state: &SharedState) -> RustBash { // Enable Python/Monty. Passing `python: true` from JS is the explicit // opt-in that must also flip the in-process Python env gate. if state.python { - if let Some(ref handler) = state.external_handler { - let h = handler.clone(); - let fn_names = state.external_functions.to_vec(); - let python_handler: PythonExternalFnHandler = Arc::new(move |name, args, kwargs| { - let h = h.clone(); - Box::pin(async move { h(name, args, kwargs).await }) - }); - builder = builder.python_with_external_handler( - profile.python_limits().clone(), - fn_names, - python_handler, - ); + builder = if state.external_functions.is_empty() { + builder.python() } else { - builder = builder.python(); - } + builder.python_with_external_handler( + profile.python_limits().clone(), + state.external_functions.clone(), + runtime_functions::python_handler(), + ) + }; builder = builder.env("BASHKIT_ALLOW_INPROCESS_PYTHON", "1"); } + if state.typescript { + builder = if state.typescript_external_functions.is_empty() { + builder.typescript() + } else { + builder.typescript_with_external_handler( + profile.typescript_limits().clone(), + state.typescript_external_functions.clone(), + runtime_functions::typescript_handler(), + ) + }; + } // Enable embedded SQLite (Turso). Passing `sqlite: true` from JS is the // explicit opt-in that must also flip the in-process SQLite env gate. @@ -3878,11 +3881,28 @@ fn build_bash_from_state(state: &SharedState) -> RustBash { } /// Build a `SharedState` from `BashOptions`, wiring up all config + interpreter. -fn shared_state_from_opts( - opts: BashOptions, - external_handler: Option, -) -> napi::Result { +fn shared_state_from_opts(opts: BashOptions) -> napi::Result { let py = opts.python.unwrap_or(false); + let ts = opts.typescript.unwrap_or(false); + let ts_fns = opts + .typescript_external_functions + .clone() + .unwrap_or_default(); + if !ts && !ts_fns.is_empty() { + return Err(napi::Error::from_reason( + "typescriptExternalFunctions requires typescript: true", + )); + } + if !py + && opts + .external_functions + .as_ref() + .is_some_and(|names| !names.is_empty()) + { + return Err(napi::Error::from_reason( + "externalFunctions requires python: true", + )); + } let sql = opts.sqlite.unwrap_or(false); let ext_fns = opts.external_functions.clone().unwrap_or_default(); let mounts = opts.mounts.clone(); @@ -3912,6 +3932,8 @@ fn shared_state_from_opts( on_output_reentry_depth: Arc::new(AtomicUsize::new(0)), in_sync_execute_depth: Arc::new(AtomicUsize::new(0)), async_execute_semaphore: Arc::new(Semaphore::new(MAX_PENDING_ASYNC_EXECUTIONS)), + async_execute_depth: Arc::new(AtomicUsize::new(0)), + sync_idle: Arc::new(tokio::sync::Notify::new()), username: opts.username.clone(), profile: opts.profile, hostname: opts.hostname.clone(), @@ -3934,10 +3956,11 @@ fn shared_state_from_opts( mounts: mounts.clone(), allowed_mount_paths: allowed_mount_paths.clone(), python: py, + typescript: ts, + typescript_external_functions: ts_fns.clone(), sqlite: sql, network: network.clone(), external_functions: ext_fns.clone(), - external_handler: external_handler.clone(), host_registry: host_registry.clone(), runtime_env: runtime_env.clone(), runtime_mounts: runtime_mounts.clone(), @@ -3968,6 +3991,8 @@ fn shared_state_from_opts( on_output_reentry_depth: tmp.on_output_reentry_depth, in_sync_execute_depth: tmp.in_sync_execute_depth, async_execute_semaphore: Arc::new(Semaphore::new(MAX_PENDING_ASYNC_EXECUTIONS)), + async_execute_depth: Arc::new(AtomicUsize::new(0)), + sync_idle: Arc::new(tokio::sync::Notify::new()), username: opts.username, profile: opts.profile, hostname: opts.hostname, @@ -3990,10 +4015,11 @@ fn shared_state_from_opts( mounts, allowed_mount_paths, python: py, + typescript: ts, + typescript_external_functions: ts_fns, sqlite: sql, network, external_functions: ext_fns, - external_handler, host_registry, runtime_env, runtime_mounts, diff --git a/crates/bashkit-js/src/runtime_functions.rs b/crates/bashkit-js/src/runtime_functions.rs new file mode 100644 index 000000000..d883c077a --- /dev/null +++ b/crates/bashkit-js/src/runtime_functions.rs @@ -0,0 +1,167 @@ +// Per-execution dispatch keeps Node callback lifetime and request context out +// of persistent snapshots. Core's revocable extension cancels pending awaits; +// the JS session aborts host I/O when native execution returns. +use std::sync::Arc; +use std::sync::atomic::Ordering; + +use bashkit::{ + ExcType, ExtFunctionResult, MontyException, PythonExternalFnHandler, RuntimeCallContext, + TypeScriptExternalFnHandler, runtime_value, +}; +use napi::JsValue; +use napi::bindgen_prelude::{Function, Promise, PromiseRaw}; +use serde::Deserialize; +use serde_json::Value; + +type RuntimeTsfn = napi::threadsafe_function::ThreadsafeFunction< + String, + Promise, + String, + napi::Status, + false, + true, +>; + +struct RuntimeSession { + dispatcher: Arc, +} + +#[derive(Deserialize)] +#[serde(tag = "status", content = "value", rename_all = "snake_case")] +enum RuntimeResponse { + Ok(Value), + Error(String), +} + +async fn dispatch(request: Value) -> Result { + let lease = RuntimeCallContext::current().execution_extension::>() + .ok_or_else(|| "host functions require execute() (async); executeSync() cannot service the JavaScript event loop".to_string())?; + let session = lease.try_with(Clone::clone).map_err(|e| e.to_string())?; + lease + .run(async { + let _permit = super::callback_semaphore() + .acquire() + .await + .map_err(|_| "host dispatcher is shutting down")?; + let promise = session + .dispatcher + .call_async(request.to_string()) + .await + .map_err(|_| "host dispatcher failed")?; + let response = promise.await.map_err(|_| "host dispatcher failed")?; + match serde_json::from_str::(&response) + .map_err(|_| "invalid host dispatcher response")? + { + RuntimeResponse::Ok(value) => { + runtime_value::validate(&value).map_err(|e| e.to_string())?; + Ok(value) + } + RuntimeResponse::Error(message) => Err(message), + } + }) + .await + .map_err(|e| e.to_string())? +} + +pub(super) fn python_handler() -> PythonExternalFnHandler { + Arc::new(|name, args, kwargs| { + Box::pin(async move { + let result = async { + let args = runtime_value::encode_python(&bashkit::MontyObject::Tuple(args)) + .map_err(|e| e.to_string())?; + let kwargs = runtime_value::encode_python(&bashkit::MontyObject::dict(kwargs)) + .map_err(|e| e.to_string())?; + let value = dispatch(serde_json::json!({ + "language": "python", "name": name, "args": args, "kwargs": kwargs + })) + .await?; + runtime_value::decode_python(&value).map_err(|e| e.to_string()) + } + .await; + match result { + Ok(value) => ExtFunctionResult::Return(value), + Err(message) => ExtFunctionResult::Error(MontyException::new( + ExcType::RuntimeError, + Some(message), + )), + } + }) + }) +} + +pub(super) fn typescript_handler() -> TypeScriptExternalFnHandler { + Arc::new(|name, args| { + Box::pin(async move { + let args = runtime_value::encode_typescript(&bashkit::ZapcodeValue::Array(args)) + .map_err(|e| e.to_string())?; + let value = + dispatch(serde_json::json!({"language":"typescript", "name":name, "args":args})) + .await?; + runtime_value::decode_typescript(&value).map_err(|e| e.to_string()) + }) + }) +} + +pub(super) fn execute<'env>( + state: Arc, + commands: String, + dispatcher: Function<'env, String, Promise>, + on_output: Option>>, +) -> napi::Result> { + super::reject_on_output_reentry(&state)?; + let env = dispatcher.value().env; + let dispatcher = Arc::new( + dispatcher + .build_threadsafe_function::() + .weak::() + .build()?, + ); + let output = on_output.map(super::create_output_tsfn).transpose()?; + let session = Arc::new(RuntimeSession { dispatcher }); + let promise = napi::bindgen_prelude::execute_tokio_future( + env, + async move { + if let Some(result) = super::validate_async_execute_input(&state, &commands) { + return Ok(result); + } + let _slot = super::acquire_async_execute_slot(&state).await?; + let mut bash = state.inner.lock().await; + let extensions = bashkit::ExecutionExtensions::new().with(session); + let cancelled = bash.cancellation_token(); + let requested = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let output = output.map(|tsfn| { + super::build_async_output_callback( + tsfn, + cancelled.clone(), + requested.clone(), + state.on_output_reentry_depth.clone(), + ) + }); + let options = bashkit::ExecOptions::new().extensions(extensions); + let options = match output { + Some((output, callback_error)) => { + // Keep the exact error slot used by the callback. + let result = bash + .exec_with_options(&commands, options.streaming(output)) + .await; + if let Some(err) = super::take_callback_error(&callback_error) { + if requested.load(Ordering::SeqCst) { + cancelled.store(false, Ordering::SeqCst); + } + return Err(err); + } + return Ok(super::js_exec_result_from_bash_result(result)); + } + None => options, + }; + Ok(super::js_exec_result_from_bash_result( + bash.exec_with_options(&commands, options).await, + )) + }, + // SAFETY: NAPI invokes this conversion on the originating JS Env. + |env, result| unsafe { + ::to_napi_value(env, result) + }, + )?; + Ok(PromiseRaw::new(env, promise)) +} diff --git a/crates/bashkit-js/wrapper.ts b/crates/bashkit-js/wrapper.ts index b5ba795fe..9c1c00fdf 100644 --- a/crates/bashkit-js/wrapper.ts +++ b/crates/bashkit-js/wrapper.ts @@ -1,3 +1,17 @@ +import { + assertRuntimeAccess, + captureRuntimeSession, + configureRuntimeFunctions, + executeNativeRuntime, + validateRuntimeFunctions, +} from "./runtime-functions.js"; +import type { RuntimeFunctionOptions } from "./runtime-functions.js"; +export type { + ExternalHandler, + TypeScriptExternalHandler, + HostFunctionContext, + RuntimeValue, +} from "./runtime-functions.js"; import { AsyncResource } from "node:async_hooks"; import type { Bash as NativeBashType, @@ -175,7 +189,7 @@ export type BuiltinCallback = (ctx: BuiltinContext) => string | Promise; /** * Options for creating a Bash or BashTool instance. */ -export interface BashOptions { +export interface BashOptions extends RuntimeFunctionOptions { /** Named resource-policy baseline. Individual limit options override it. */ profile?: ExecutionProfileName; username?: string; @@ -508,6 +522,7 @@ function toNativeOptions( options?: BashOptions, resolvedFiles?: Record, ): NativeBashOptions | undefined { + validateRuntimeFunctions(options); if (!options && !resolvedFiles) return undefined; return { username: options?.username, @@ -530,6 +545,8 @@ function toNativeOptions( allowedMountPaths: options?.allowedMountPaths, python: options?.python, externalFunctions: options?.externalFunctions, + typescript: options?.typescript, + typescriptExternalFunctions: options?.typescriptExternalFunctions, sqlite: options?.sqlite, network: options?.network, }; @@ -569,16 +586,25 @@ export interface FileSystemRealOptions { } export class FileSystem { - private native: any; + private nativeValue: any; + private owner?: object; + private get native(): any { + if (this.owner) assertRuntimeAccess(this.owner); + return this.nativeValue; + } + private set native(value: any) { + this.nativeValue = value; + } private external?: unknown; constructor() { this.native = nativeCreateFileSystem(); } - static fromNative(nativeFs: any): FileSystem { + static fromNative(nativeFs: any, owner?: object): FileSystem { const fs = Object.create(FileSystem.prototype) as FileSystem; fs.native = nativeFs; + fs.owner = owner; fs.external = undefined; return fs; } @@ -760,6 +786,7 @@ export class Bash { const resolved = resolveFilesSync(options?.files); this.native = new NativeBash(toNativeOptions(options, resolved)); this.maxInputBytes = options?.maxInputBytes ?? DEFAULT_MAX_INPUT_BYTES; + this.native = configureRuntimeFunctions(this.native, options); registerCustomBuiltins(this.native, options?.customBuiltins); } @@ -782,6 +809,7 @@ export class Bash { const instance = Object.create(Bash.prototype) as Bash; instance.native = new NativeBash(toNativeOptions(options, resolved)); instance.maxInputBytes = options?.maxInputBytes ?? DEFAULT_MAX_INPUT_BYTES; + instance.native = configureRuntimeFunctions(instance.native, options); registerCustomBuiltins(instance.native, options?.customBuiltins); return instance; } @@ -886,6 +914,7 @@ export class Bash { commands: string, options?: ExecuteOptions, ): Promise { + const runtimeSession = captureRuntimeSession(this.native); const nativeOnOutput = toNativeOnOutput(options?.onOutput); const signal = options?.signal; if (signal?.aborted) { @@ -910,13 +939,12 @@ export class Bash { }; signal.addEventListener("abort", onAbort, { once: true }); try { - if (nativeOnOutput) { - return await this.native.executeWithOutput( - commands, - nativeOnOutput, - ); - } - return await this.native.execute(commands); + return await executeNativeRuntime( + this.native, + commands, + nativeOnOutput, + runtimeSession, + ); } finally { signal.removeEventListener("abort", onAbort); if (signalTriggered) { @@ -924,11 +952,13 @@ export class Bash { } } } - if (nativeOnOutput) { - return this.native.executeWithOutput(commands, nativeOnOutput); - } - return this.native.execute(commands); - }); + return executeNativeRuntime( + this.native, + commands, + nativeOnOutput, + runtimeSession, + ); + }).finally(() => runtimeSession?.finish()); } /** @@ -1127,25 +1157,42 @@ export class Bash { * const restored = Bash.fromSnapshot(snapshot); * ``` */ - static fromSnapshot(data: Uint8Array, options?: SnapshotOptions): Bash { - const instance = new Bash(); + static fromSnapshot( + data: Uint8Array, + snapshotOptions?: SnapshotOptions, + options?: BashOptions, + ): Bash { + const instance = Object.create(Bash.prototype) as Bash; + const resolved = resolveFilesSync(options?.files); instance.native = NativeBash.fromSnapshot( Buffer.from(data), - undefined, - toNativeSnapshotOptions(options), + toNativeOptions(options, resolved), + toNativeSnapshotOptions(snapshotOptions), ); + instance.native = configureRuntimeFunctions(instance.native, options); + registerCustomBuiltins(instance.native, options?.customBuiltins); + instance.maxInputBytes = options?.maxInputBytes ?? DEFAULT_MAX_INPUT_BYTES; return instance; } /** * Create a new Bash instance from a HMAC-protected snapshot. */ - static fromSnapshotKeyed(data: Uint8Array, key: Uint8Array): Bash { - const instance = new Bash(); + static fromSnapshotKeyed( + data: Uint8Array, + key: Uint8Array, + options?: BashOptions, + ): Bash { + const instance = Object.create(Bash.prototype) as Bash; + const resolved = resolveFilesSync(options?.files); instance.native = NativeBash.fromSnapshotKeyed( Buffer.from(data), Buffer.from(key), + toNativeOptions(options, resolved), ); + instance.native = configureRuntimeFunctions(instance.native, options); + registerCustomBuiltins(instance.native, options?.customBuiltins); + instance.maxInputBytes = options?.maxInputBytes ?? DEFAULT_MAX_INPUT_BYTES; return instance; } @@ -1234,7 +1281,7 @@ export class Bash { /** Get a FileSystem handle for direct VFS operations. */ fs(): FileSystem { - return FileSystem.fromNative(this.native.fs()); + return FileSystem.fromNative(this.native.fs(), this.native); } mount(vfsPath: string, fs: FileSystem, readOnly?: boolean): void; @@ -1337,6 +1384,7 @@ export class BashTool { const resolved = resolveFilesSync(options?.files); this.native = new NativeBashTool(toNativeOptions(options, resolved)); this.maxInputBytes = options?.maxInputBytes ?? DEFAULT_MAX_INPUT_BYTES; + this.native = configureRuntimeFunctions(this.native, options); registerCustomBuiltins(this.native, options?.customBuiltins); } @@ -1348,6 +1396,7 @@ export class BashTool { const instance = Object.create(BashTool.prototype) as BashTool; instance.native = new NativeBashTool(toNativeOptions(options, resolved)); instance.maxInputBytes = options?.maxInputBytes ?? DEFAULT_MAX_INPUT_BYTES; + instance.native = configureRuntimeFunctions(instance.native, options); registerCustomBuiltins(instance.native, options?.customBuiltins); return instance; } @@ -1413,6 +1462,7 @@ export class BashTool { commands: string, options?: ExecuteOptions, ): Promise { + const runtimeSession = captureRuntimeSession(this.native); const nativeOnOutput = toNativeOnOutput(options?.onOutput); const signal = options?.signal; if (signal?.aborted) { @@ -1437,13 +1487,12 @@ export class BashTool { }; signal.addEventListener("abort", onAbort, { once: true }); try { - if (nativeOnOutput) { - return await this.native.executeWithOutput( - commands, - nativeOnOutput, - ); - } - return await this.native.execute(commands); + return await executeNativeRuntime( + this.native, + commands, + nativeOnOutput, + runtimeSession, + ); } finally { signal.removeEventListener("abort", onAbort); if (signalTriggered) { @@ -1451,11 +1500,13 @@ export class BashTool { } } } - if (nativeOnOutput) { - return this.native.executeWithOutput(commands, nativeOnOutput); - } - return this.native.execute(commands); - }); + return executeNativeRuntime( + this.native, + commands, + nativeOnOutput, + runtimeSession, + ); + }).finally(() => runtimeSession?.finish()); } /** @@ -1595,6 +1646,7 @@ export class BashTool { toNativeOptions(options, resolved), toNativeSnapshotOptions(snapshotOptions), ); + instance.native = configureRuntimeFunctions(instance.native, options); registerCustomBuiltins(instance.native, options?.customBuiltins); instance.maxInputBytes = options?.maxInputBytes ?? DEFAULT_MAX_INPUT_BYTES; return instance; @@ -1615,6 +1667,7 @@ export class BashTool { Buffer.from(key), toNativeOptions(options, resolved), ); + instance.native = configureRuntimeFunctions(instance.native, options); registerCustomBuiltins(instance.native, options?.customBuiltins); instance.maxInputBytes = options?.maxInputBytes ?? DEFAULT_MAX_INPUT_BYTES; return instance; @@ -1729,7 +1782,7 @@ export class BashTool { /** Get a FileSystem handle for direct VFS operations. */ fs(): FileSystem { - return FileSystem.fromNative(this.native.fs()); + return FileSystem.fromNative(this.native.fs(), this.native); } mount(vfsPath: string, fs: FileSystem, readOnly?: boolean): void; diff --git a/crates/bashkit/docs/python.md b/crates/bashkit/docs/python.md index efd5f6b3c..dc2958c78 100644 --- a/crates/bashkit/docs/python.md +++ b/crates/bashkit/docs/python.md @@ -200,6 +200,8 @@ let help = tool.help(); // Includes a Markdown Notes section with Python hints The builtin's `llm_hint()` is automatically included in the tool's documentation, so LLMs know file I/O is VFS-scoped and HTTP requests/classes are unavailable. +`BashTool::builder().python_with_external_handler(limits, names, handler)` exposes the same named callbacks through the Rust tool builder. The Python runtime gate still applies. Foreign bindings can use `bashkit::runtime_value` for bounded JSON conversion; native Rust callbacks keep their Monty value types. + ## Limitations **VFS-only file I/O.** `open()` and `pathlib.Path` read/write Bashkit's virtual diff --git a/crates/bashkit/src/lib.rs b/crates/bashkit/src/lib.rs index e7c7cf243..6a156827b 100644 --- a/crates/bashkit/src/lib.rs +++ b/crates/bashkit/src/lib.rs @@ -456,6 +456,8 @@ pub mod parser; mod profile; #[cfg(any(feature = "python", feature = "typescript"))] mod runtime_call; +#[cfg(any(feature = "python", feature = "typescript"))] +pub mod runtime_value; mod snapshot; mod stream; #[cfg(feature = "terminal")] diff --git a/crates/bashkit/src/runtime_value.rs b/crates/bashkit/src/runtime_value.rs new file mode 100644 index 000000000..53d105b56 --- /dev/null +++ b/crates/bashkit/src/runtime_value.rs @@ -0,0 +1,220 @@ +//! Bounded JSON values at foreign host-function boundaries. +//! +//! Native Rust handlers retain their language-specific value APIs. Bindings +//! using JSON share this explicit subset instead of stringifying unsupported +//! objects or serializing VM implementation details. Hosts own I/O and service +//! authorization; conversion does not grant further sandbox capabilities. + +use serde_json::Value; + +use crate::{Error, Result}; + +/// Maximum nesting across a foreign host-function boundary. +pub const MAX_DEPTH: usize = 64; + +fn depth_guard(depth: usize) -> Result<()> { + if depth > MAX_DEPTH { + return Err(Error::Execution( + "host value nesting depth exceeds maximum of 64".into(), + )); + } + Ok(()) +} + +/// Validate the shared JSON subset (including signed 64-bit integer bounds). +pub fn validate(value: &Value) -> Result<()> { + fn walk(value: &Value, depth: usize) -> Result<()> { + depth_guard(depth)?; + match value { + Value::Number(n) if n.is_u64() && n.as_i64().is_none() => Err(Error::Execution( + "host integers must fit in signed 64 bits".into(), + )), + Value::Array(values) => values.iter().try_for_each(|v| walk(v, depth + 1)), + Value::Object(values) => values.values().try_for_each(|v| walk(v, depth + 1)), + _ => Ok(()), + } + } + walk(value, 0) +} + +#[cfg(any(feature = "python", feature = "typescript"))] +fn number(value: f64) -> Result { + serde_json::Number::from_f64(value) + .map(Value::Number) + .ok_or_else(|| Error::Execution("host numbers must be finite".into())) +} + +/// Encode a Python value as JSON, rejecting unsupported objects. +#[cfg(feature = "python")] +pub fn encode_python(value: &crate::MontyObject) -> Result { + fn encode(value: &crate::MontyObject, depth: usize) -> Result { + use crate::MontyObject as M; + depth_guard(depth)?; + Ok(match value { + M::None => Value::Null, + M::Bool(v) => Value::Bool(*v), + M::Int(v) => Value::from(*v), + M::BigInt(v) => Value::from(v.to_string().parse::().map_err(|_| { + Error::Execution("host integers must fit in signed 64 bits".into()) + })?), + M::Float(v) => number(*v)?, + M::String(v) => Value::String(v.clone()), + M::List(values) | M::Tuple(values) => Value::Array( + values + .iter() + .map(|v| encode(v, depth + 1)) + .collect::>()?, + ), + M::Dict(values) => { + let mut object = serde_json::Map::new(); + for (key, value) in values { + let M::String(key) = key else { + return Err(Error::Execution( + "host dictionaries require string keys".into(), + )); + }; + object.insert(key.clone(), encode(value, depth + 1)?); + } + Value::Object(object) + } + _ => { + return Err(Error::Execution( + "Python host arguments must be JSON values".into(), + )); + } + }) + } + encode(value, 0) +} + +/// Decode JSON into the Python host-function value subset. +#[cfg(feature = "python")] +pub fn decode_python(value: &Value) -> Result { + fn decode(value: &Value) -> crate::MontyObject { + use crate::MontyObject as M; + match value { + Value::Null => M::None, + Value::Bool(v) => M::Bool(*v), + Value::Number(v) => match v.as_i64() { + Some(v) => M::Int(v), + None => M::Float(v.as_f64().expect("validated JSON number")), + }, + Value::String(v) => M::String(v.clone()), + Value::Array(values) => M::List(values.iter().map(decode).collect()), + Value::Object(values) => M::dict( + values + .iter() + .map(|(k, v)| (M::String(k.clone()), decode(v))) + .collect::>(), + ), + } + } + validate(value)?; + Ok(decode(value)) +} + +/// Encode a TypeScript value as JSON; undefined maps to null. +#[cfg(feature = "typescript")] +pub fn encode_typescript(value: &crate::ZapcodeValue) -> Result { + fn encode(value: &crate::ZapcodeValue, depth: usize) -> Result { + use crate::ZapcodeValue as Z; + depth_guard(depth)?; + Ok(match value { + Z::Undefined | Z::Null => Value::Null, + Z::Bool(v) => Value::Bool(*v), + Z::Int(v) => Value::from(*v), + Z::Float(v) => number(*v)?, + Z::String(v) => Value::String(v.to_string()), + Z::Array(values) => Value::Array( + values + .iter() + .map(|v| encode(v, depth + 1)) + .collect::>()?, + ), + Z::Object(values) => Value::Object( + values + .iter() + .map(|(k, v)| Ok((k.to_string(), encode(v, depth + 1)?))) + .collect::>()?, + ), + _ => { + return Err(Error::Execution( + "TypeScript host arguments must be JSON values".into(), + )); + } + }) + } + encode(value, 0) +} + +/// Decode JSON into the TypeScript host-function value subset. +#[cfg(feature = "typescript")] +pub fn decode_typescript(value: &Value) -> Result { + fn decode(value: &Value) -> crate::ZapcodeValue { + use crate::ZapcodeValue as Z; + use std::sync::Arc; + match value { + Value::Null => Z::Null, + Value::Bool(v) => Z::Bool(*v), + Value::Number(v) => match v.as_i64() { + Some(v) => Z::Int(v), + None => Z::Float(v.as_f64().expect("validated JSON number")), + }, + Value::String(v) => Z::String(Arc::from(v.as_str())), + Value::Array(values) => Z::Array(values.iter().map(decode).collect()), + Value::Object(values) => Z::Object( + values + .iter() + .map(|(k, v)| (Arc::from(k.as_str()), decode(v))) + .collect(), + ), + } + } + validate(value)?; + Ok(decode(value)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_deep_values_and_unsigned_overflow() { + let mut nested = Value::Null; + for _ in 0..MAX_DEPTH + 2 { + nested = Value::Array(vec![nested]); + } + assert!(validate(&nested).is_err()); + assert!(validate(&Value::from(u64::MAX)).is_err()); + assert!(validate(&Value::from(i64::MAX)).is_ok()); + } + + #[cfg(feature = "python")] + #[test] + fn python_roundtrip_and_unsupported_values() { + use crate::MontyObject as M; + let value = serde_json::json!({"items": [null, true, 42, 0.5, "text"]}); + assert_eq!( + encode_python(&decode_python(&value).unwrap()).unwrap(), + value + ); + assert!(encode_python(&M::Float(f64::NAN)).is_err()); + assert!(encode_python(&M::Bytes(vec![1])).is_err()); + assert!(encode_python(&M::dict(vec![(M::Int(1), M::None)])).is_err()); + } + + #[cfg(feature = "typescript")] + #[test] + fn typescript_roundtrip_and_undefined() { + let value = serde_json::json!({"items": [null, true, 42, 0.5, "text"]}); + assert_eq!( + encode_typescript(&decode_typescript(&value).unwrap()).unwrap(), + value + ); + assert_eq!( + encode_typescript(&crate::ZapcodeValue::Undefined).unwrap(), + Value::Null + ); + assert!(encode_typescript(&crate::ZapcodeValue::Float(f64::INFINITY)).is_err()); + } +} diff --git a/crates/bashkit/src/tool.rs b/crates/bashkit/src/tool.rs index fa25f507f..30ed711d5 100644 --- a/crates/bashkit/src/tool.rs +++ b/crates/bashkit/src/tool.rs @@ -711,6 +711,31 @@ impl BashToolBuilder { .builtin("python3", Box::new(Python::with_limits(limits))) } + /// Enable embedded Python with named asynchronous host functions. + /// + /// The handler is shared across executions; request-local state belongs in + /// [`crate::RuntimeCallContext`]. The Python runtime gate still applies. + #[cfg(feature = "python")] + pub fn python_with_external_handler( + self, + limits: crate::builtins::PythonLimits, + external_fns: Vec, + handler: crate::builtins::PythonExternalFnHandler, + ) -> Self { + use crate::builtins::Python; + self.builtin( + "python", + Box::new( + Python::with_limits(limits.clone()) + .with_external_handler(external_fns.clone(), handler.clone()), + ), + ) + .builtin( + "python3", + Box::new(Python::with_limits(limits).with_external_handler(external_fns, handler)), + ) + } + /// Enable embedded TypeScript/JavaScript execution via ZapCode with defaults. /// /// Requires the `typescript` feature flag. @@ -1712,6 +1737,69 @@ mod tests { assert_eq!(output.result["stdout"], "4\n"); } + #[cfg(feature = "python")] + #[tokio::test] + async fn test_python_host_functions_execute_through_tool() { + let handler: crate::PythonExternalFnHandler = Arc::new(|name, args, kwargs| { + Box::pin(async move { + assert_eq!(name, "lookup"); + assert_eq!(args, vec![crate::MontyObject::Int(21)]); + assert_eq!(kwargs.len(), 1); + crate::ExtFunctionResult::Return(crate::MontyObject::Int(42)) + }) + }); + let tool = BashTool::builder() + .env("BASHKIT_ALLOW_INPROCESS_PYTHON", "1") + .python_with_external_handler( + crate::PythonLimits::default(), + vec!["lookup".into()], + handler, + ) + .build(); + for command in ["python", "python3"] { + let mut bash = tool.create_bash(); + let result = bash + .exec(&format!("{command} -c 'print(lookup(21, multiplier=2))'")) + .await + .unwrap(); + assert_eq!(result.stdout, "42\n", "{}", result.stderr); + let unknown = bash + .exec(&format!("{command} -c 'unregistered()'")) + .await + .unwrap(); + assert_ne!(unknown.exit_code, 0); + } + } + + #[cfg(feature = "typescript")] + #[tokio::test] + async fn test_typescript_host_functions_execute_through_tool() { + let handler: crate::TypeScriptExternalFnHandler = Arc::new(|name, args| { + Box::pin(async move { + assert_eq!(name, "lookup"); + assert_eq!(args, vec![crate::ZapcodeValue::Int(21)]); + Ok(crate::ZapcodeValue::Int(42)) + }) + }); + let tool = BashTool::builder() + .typescript_with_external_handler( + crate::TypeScriptLimits::default(), + vec!["lookup".into()], + handler, + ) + .build(); + let mut bash = tool.create_bash(); + let result = bash.exec("ts -c 'await lookup(21)'").await.unwrap(); + assert_eq!(result.stdout, "42\n", "{}", result.stderr); + assert_ne!( + bash.exec("ts -c 'await unregistered()'") + .await + .unwrap() + .exit_code, + 0 + ); + } + #[cfg(feature = "http_client")] #[tokio::test] async fn test_configure_forwards_network_policy_and_transport() { diff --git a/docs/start-node.md b/docs/start-node.md index ce05323fb..bd148f008 100644 --- a/docs/start-node.md +++ b/docs/start-node.md @@ -21,6 +21,49 @@ const result = bash.executeSync('echo "Hello, World!"'); console.log(result.stdout); ``` +## Python and TypeScript host functions + +Enable either embedded runtime and register named host functions. The same options +work with `Bash`, `BashTool`, and the LangChain, OpenAI, Anthropic and Vercel AI adapters. + +```typescript +const bash = new Bash({ + python: true, + externalFunctions: ["lookup"], + externalHandler: async (name, args, kwargs, { signal }) => { + const response = await fetch(`https://api.example.com/items/${args[0]}`, { signal }); + return response.json(); + }, + typescript: true, + typescriptExternalFunctions: ["double"], + typescriptExternalHandler: (_name, args) => Number(args[0]) * 2, +}); +await bash.execute(`python -c 'print(lookup(21))'`); +console.log((await bash.execute(`ts -c 'await double(21)'`)).stdout); // 42 +``` + +Use `execute()` for scripts that invoke JavaScript handlers. `executeSync()` +returns a script error for such calls because it blocks the JavaScript event loop. +Handlers may return a value or a promise. Python receives positional arguments and +keyword arguments separately; TypeScript receives positional arguments. + +Values use the JSON subset: null, booleans, finite numbers, strings, arrays, and +objects with string keys, with nesting limited to 64. Integral values must fit +JavaScript's safe integer range. Python tuples become arrays; TypeScript +`undefined` and an omitted host return become null. Bytes, functions, arbitrary +class instances and cyclic values produce script errors. + +Each execution captures the caller's `AsyncLocalStorage` context before queuing. +Its handler signal aborts on cancellation, timeout or completion; pass it to host +I/O. A callback cannot reenter its own interpreter or live filesystem. Use another +instance for nested execution. Independent asynchronous requests can share an +instance and queue safely. Synchronous access during asynchronous execution fails +promptly. Callbacks survive reset; supply the options again when restoring a +snapshot, since host code and request context are never serialized. + +Host functions run trusted application code. Enforce service authorization and +I/O limits there; registering a function explicitly grants scripts that capability. + ## Persistent state A `Bash` instance keeps its environment and virtual filesystem across calls: diff --git a/examples/runtime_functions.mjs b/examples/runtime_functions.mjs new file mode 100644 index 000000000..1c42de822 --- /dev/null +++ b/examples/runtime_functions.mjs @@ -0,0 +1,21 @@ +// Named host functions let sandboxed Python/TypeScript call application services. +import assert from "node:assert/strict"; +import { BashTool } from "@everruns/bashkit"; + +const bash = new BashTool({ + python: true, + externalFunctions: ["lookup"], + externalHandler: async (_name, args, kwargs, { signal }) => { + assert.equal(signal.aborted, false); + return { value: Number(args[0]) * Number(kwargs.multiplier) }; + }, + typescript: true, + typescriptExternalFunctions: ["double"], + typescriptExternalHandler: (_name, args) => Number(args[0]) * 2, +}); +assert.equal( + (await bash.execute(`python -c 'print(lookup(21, multiplier=2)["value"])'`)).stdout, + "42\n", +); +assert.equal((await bash.execute(`ts -c 'await double(21)'`)).stdout, "42\n"); +console.log("Python and TypeScript host functions: 42"); diff --git a/site/src/content/apidocs/typescript.md b/site/src/content/apidocs/typescript.md index 91bea5b86..a7c3f9c82 100644 --- a/site/src/content/apidocs/typescript.md +++ b/site/src/content/apidocs/typescript.md @@ -440,7 +440,7 @@ const bash = await Bash.create({ ### `fromSnapshot` ```typescript -Bash.fromSnapshot(data: Uint8Array, options?: SnapshotOptions): Bash +Bash.fromSnapshot(data: Uint8Array, snapshotOptions?: SnapshotOptions, options?: BashOptions): Bash ``` Create a new Bash instance from a snapshot. @@ -453,7 +453,7 @@ const restored = Bash.fromSnapshot(snapshot); ### `fromSnapshotKeyed` ```typescript -Bash.fromSnapshotKeyed(data: Uint8Array, key: Uint8Array): Bash +Bash.fromSnapshotKeyed(data: Uint8Array, key: Uint8Array, options?: BashOptions): Bash ``` Create a new Bash instance from a HMAC-protected snapshot. @@ -985,7 +985,7 @@ FileSystem.fromExternal(external: unknown): FileSystem ### `fromNative` ```typescript -FileSystem.fromNative(nativeFs: any): FileSystem +FileSystem.fromNative(nativeFs: any, owner?: object): FileSystem ``` ### `real` @@ -1331,6 +1331,7 @@ Options for creating a Bash or BashTool instance. These function names become available as Python builtins within the embedded interpreter. When called, they invoke the external handler. +- **`externalHandler?`** — `ExternalHandler` - **`files?`** — `Record` Files to mount in the virtual filesystem. @@ -1389,6 +1390,9 @@ Options for creating a Bash or BashTool instance. When set, commands that exceed this duration are aborted with exit code 124 (matching the bash `timeout` convention). +- **`typescript?`** — `boolean` +- **`typescriptExternalFunctions?`** — `string[]` +- **`typescriptExternalHandler?`** — `TypeScriptExternalHandler` - **`username?`** — `string` ## BuiltinContext @@ -1504,6 +1508,14 @@ Standalone filesystem handle for direct VFS operations and native addon interop. - **`allowedMountPaths`** — `string[]` - **`writable?`** — `boolean` +## HostFunctionContext + +### Fields + +- **`signal`** — `AbortSignal` + + Aborted when execution is cancelled, times out, or completes. + ## NetworkCredential Credential injected into outbound HTTP requests matching `pattern`. @@ -1869,6 +1881,12 @@ type CheckoutPolicy = "strict" | "superset" | "force" type ExecutionProfileName = typeof ExecutionProfile[keyof typeof ExecutionProfile] ``` +## ExternalHandler + +```typescript +type ExternalHandler = (name: string, args: RuntimeValue[], kwargs: Record, context: HostFunctionContext) => RuntimeValue | undefined | Promise +``` + ## FileValue A file value: either a string, a sync function returning a string, @@ -1886,6 +1904,18 @@ type FileValue = string | () => string | () => Promise type OnOutput = (chunk: OutputChunk) => void ``` +## RuntimeValue + +```typescript +type RuntimeValue = null | boolean | number | string | RuntimeValue[] | object +``` + +## TypeScriptExternalHandler + +```typescript +type TypeScriptExternalHandler = (name: string, args: RuntimeValue[], context: HostFunctionContext) => RuntimeValue | undefined | Promise +``` + --- # Subpath modules @@ -2263,6 +2293,7 @@ Options for configuring the bash tool adapter. These function names become available as Python builtins within the embedded interpreter. When called, they invoke the external handler. +- **`externalHandler?`** — `ExternalHandler` - **`files?`** — `Record` Pre-populate VFS files. Keys are absolute paths, values are file contents. @@ -2336,6 +2367,9 @@ Options for configuring the bash tool adapter. Commands exceeding this duration are aborted with exit code 124. Framework-level timeouts can be propagated here to ensure bashkit stops execution when the framework cancels a tool call. +- **`typescript?`** — `boolean` +- **`typescriptExternalFunctions?`** — `string[]` +- **`typescriptExternalHandler?`** — `TypeScriptExternalHandler` - **`username?`** — `string` ### HandlerOptions @@ -2488,6 +2522,7 @@ Options for configuring the bash tool adapter. These function names become available as Python builtins within the embedded interpreter. When called, they invoke the external handler. +- **`externalHandler?`** — `ExternalHandler` - **`files?`** — `Record` Pre-populate VFS files. Keys are absolute paths, values are file contents. @@ -2557,6 +2592,9 @@ Options for configuring the bash tool adapter. Commands exceeding this duration are aborted with exit code 124. Framework-level timeouts can be propagated here to ensure bashkit stops execution when the framework cancels a tool call. +- **`typescript?`** — `boolean` +- **`typescriptExternalFunctions?`** — `string[]` +- **`typescriptExternalHandler?`** — `TypeScriptExternalHandler` - **`username?`** — `string` ### HandlerOptions @@ -2694,6 +2732,7 @@ Options for configuring the bash tool adapter. These function names become available as Python builtins within the embedded interpreter. When called, they invoke the external handler. +- **`externalHandler?`** — `ExternalHandler` - **`files?`** — `Record` Pre-populate VFS files. Keys are absolute paths, values are file contents. @@ -2748,4 +2787,7 @@ Options for configuring the bash tool adapter. When set, commands that exceed this duration are aborted with exit code 124 (matching the bash `timeout` convention). +- **`typescript?`** — `boolean` +- **`typescriptExternalFunctions?`** — `string[]` +- **`typescriptExternalHandler?`** — `TypeScriptExternalHandler` - **`username?`** — `string` From 9aafd63d6f02c19833539a0c60b0218eea5e4098 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Fri, 16 Oct 2026 19:48:03 +0000 Subject: [PATCH 2/5] chore(knowledge): track runtime callback parity and Node execution rules --- AGENTS.md | 1 + knowledge/log.md | 2 ++ knowledge/runtimes/index.md | 1 + knowledge/runtimes/node-package.md | 50 +++++++++++++++++++++++++++ knowledge/security/threat-model.md | 2 +- knowledge/status/capability-parity.md | 13 +++++-- 6 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 knowledge/runtimes/node-package.md diff --git a/AGENTS.md b/AGENTS.md index 04cc444fc..ba8c8322d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,6 +44,7 @@ Fix root cause. Unsure: read more code; if stuck, ask w/ short options. Unrecogn | operations/eval | LLM eval study on the mira framework, dataset format, scoring | | operations/oils-spec | Upstream Oils spec suite pass rate vs real bash (`just oils-spec`) | | operations/maintenance | Pre-release maintenance requirements | +| runtimes/node-package | Native JavaScript package, runtime callbacks, caller context and worker-thread safety | | runtimes/python-package | Python package, PyPI wheels, platform matrix | | integrations/scripted-tool-orchestration | Compose ToolDef+callback pairs into OrchestratorTool via bash scripts | | integrations/ssh-support | Sandboxed SSH/SCP/SFTP operations | diff --git a/knowledge/log.md b/knowledge/log.md index 1102284ab..68d565981 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,8 @@ ## 2026-10-16 +* **Node runtime parity**: Added Python and TypeScript named host callbacks to Bash/BashTool and JS agent adapters, with per-execution caller context, cooperative abort, reentry guards and worker/platform tests. Added explicit runtime host-function capability rows and a shared bounded JSON codec for foreign bindings. See [Node Package](runtimes/node-package.md). + * **Python / agent integrations**: Added Python and TypeScript host function callbacks to `BashTool`, LangChain and deepagents. Reuse per-execution callback sessions for caller-loop affinity, ContextVars and cancellation; sync execution uses the dedicated private worker loop. Reentry is callback-context-specific so independent requests queue safely. Private-loop timeout cancellation now targets its own work item; native result deliveries drain before interpreter finalization to prevent a shutdown race. Added FastAPI, real Jupyter, alternate-loop and cross-thread regression coverage plus macOS/Windows callback CI. See [Python Package](runtimes/python-package.md). ## 2026-10-15 diff --git a/knowledge/runtimes/index.md b/knowledge/runtimes/index.md index 1408ac4bf..a605bb988 100644 --- a/knowledge/runtimes/index.md +++ b/knowledge/runtimes/index.md @@ -11,3 +11,4 @@ * [Browser Package](browser-package.md) - Slim single-threaded WebAssembly package design for browsers and JavaScript runtimes. * [Non-JS WebAssembly Embedding](non-js-wasm.md) - Running bashkit as a wasm component with no JS engine and no WASI, including the Hyperlight micro-VM guest. * [C API](c-api.md) - Versioned native C ABI, ownership rules, packaging, and compatibility boundaries. +* [Node Package](node-package.md) - Native JavaScript bindings and embedded-runtime callback execution across Node, Bun and Deno. diff --git a/knowledge/runtimes/node-package.md b/knowledge/runtimes/node-package.md new file mode 100644 index 000000000..6d9545fcc --- /dev/null +++ b/knowledge/runtimes/node-package.md @@ -0,0 +1,50 @@ +--- +type: Package Design +title: Node Package +description: Native JavaScript bindings and embedded-runtime callback execution across Node, Bun and Deno. +tags: + - bashkit + - javascript + - callbacks +--- + +# Node package host functions + +`@everruns/bashkit` enables Python and TypeScript independently. Both `Bash` +and `BashTool` accept runtime names and dispatchers; all JS agent adapters +forward these options. LangChain awaits execution and forwards RunnableConfig's +AbortSignal. Python dispatch includes kwargs; TypeScript uses positional values. + +Foreign runtime values use the shared core `runtime_value` JSON codec: depth 64, +finite numbers, signed 64-bit integers, string-key dictionaries, arrays and scalars. +The JavaScript boundary additionally rejects integers outside the safe range. +Unsupported values fail instead of stringifying. Undefined maps to null. +Native Rust callbacks retain the language-specific value APIs. + +Each async execution binds an AsyncResource before entering the bounded JS queue. +This preserves caller AsyncLocalStorage across NAPI and queued requests. Its +revocable execution extension owns a weak NAPI threadsafe dispatcher. Completion, +timeout or cancellation drops native waits and aborts the JS session's signal. +Host I/O must observe that signal; arbitrary application side effects cannot be +rolled back. Request context and callbacks are never snapshot data. + +An AsyncLocalStorage owner marker rejects same-instance callback reentry, +including live filesystem handles, while independent async requests queue. Native +async-execution and sync-entry counters close the race where synchronous access +blocks the loop just as a handler or streaming output callback starts. Blocking +access fails during asynchronous execution; asynchronous entry waits for any +earlier synchronous entry to finish. Sync host calls fail explicitly; +ordinary synchronous commands remain available. Separate worker-thread environments +own separate dispatchers; weak references do not keep Node alive at shutdown. + +`runtime-functions.spec.ts` exercises real HTTP request contexts, aborts, timeout, +queued calls, worker threads, adapters, reentry, bad values and snapshot/reset. +`runtime-compat/runtime-functions.test.mjs` runs in Node, Bun and Deno. Dedicated +macOS and Windows CI runs the callback suite as well as Linux's runtime matrix. + +## See also + +- [Python Package](python-package.md) +- [Browser Package](browser-package.md) +- [Capability Parity](../status/capability-parity.md) +- [Threat Model](../security/threat-model.md) diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 33465973d..5ee71b09b 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -470,7 +470,7 @@ registered non-shell builtin runs that builtin on the script path (same sandbox | TM-INF-025 | Untrusted generated Rust runs in drift CI with repository write token | The coreutils argument-drift workflow regenerates Rust from third-party `uutils/coreutils` `uu_app()` then builds/tests bashkit; if the generator preserved arbitrary upstream statements and the job held `contents: write`/`pull-requests: write`, malicious upstream code could execute with repository write impact | Two-layer fix: (1) `bashkit-coreutils-port` validates args-mode `uu_app()` before emission, accepts only a single tail clap `Command` builder chain, or `let = Command::new(...); .(...)` with the tail chained off the let-bound ident; both shapes pass a disallowed-method/closure/macro/block visitor. Regression tests `rejects_executable_statements_in_uu_app`, `rejects_let_with_non_command_initializer`, `rejects_tail_not_chained_off_let_binding`, `rejects_three_statement_body`. (2) `.github/workflows/coreutils-args-drift.yml` separates privilege: regeneration/build/test runs with `contents: read` + `persist-credentials: false`; the `open-pr` job has write permission but never builds or executes generated Rust, only commits the tested artifact | **FIXED** | | TM-INF-026 | Fork-PR secret exfiltration via approved CI run | GitHub's first-time-contributor gate runs the workflow **from the PR head**, and approved contributors auto-run thereafter. The `examples` job in `ci.yml` set `DOPPLER_TOKEN` at job level and `ANTHROPIC_API_KEY` at step level, so a fork PR could edit any `examples/*.rs`/`build.rs`/script the job runs and exfiltrate. `DOPPLER_TOKEN` is the master key, it fetches every other secret in the Doppler config, so one leak compromises all. Every `doppler run --` also injected the entire config into the step env | Trusted-run gates exclude PR execution from secret-backed examples. Dedicated credential-fetch steps receive `DOPPLER_TOKEN`, request one API key, mask it, and exit before build/example processes run. Execution steps receive only the requested API key through step output/env; Docker inherits it by name, without putting its value in argv. No broad token remains in the execution shell or a live Doppler parent. This is process-lifetime isolation on a trusted runner, not a sandbox against compromised dependencies persisting across steps. Regression tests execute the workflow scripts and inspect child/parent environments: `scripts/tests/test_maintenance_security.py`. CI has contents-read permission only and checkout credentials are not persisted anywhere they could reach untrusted code (TM-INF-034). Release examples install the committed dependency lockfile with lifecycle scripts disabled before linking the built artifact; `scripts/tests/test_ci_supply_chain.py` enforces both boundaries | **FIXED** | | TM-INF-027 | Package publish from unverified release refs | A user with Actions/write privileges could dispatch release/publish workflows from a branch or unprotected `vX.Y.Z` tag; `release.yml` dispatches downstream workflows with `--ref "$TAG"`, and `publish-js.yml` treated stable `package.json` semver as `latest` even on non-tag refs, an attacker-controlled ref could source official crates.io/npm/PyPI/CLI/Homebrew artifacts | Release dispatches require full ref `refs/heads/main`; `release.yml` verifies the source is reachable from `origin/main` and the created tag resolves to the current `GITHUB_SHA` before dispatching; `publish.yml` rejects manual runs unless `GITHUB_REF` is a stable release tag matching `Cargo.toml` on a main-reachable commit; `publish-js.yml` requires main-reachable commit, assigns npm `latest` only for stable `vX.Y.Z` tag refs matching `package.json`; `publish-python.yml` applies the same main-reachable stable-tag + Cargo version gates; `cli-binaries.yml` accepts only stable `vX.Y.Z` inputs and verifies workflow ref = tag = `Cargo.toml` version on a main-reachable commit before building/uploading or updating Homebrew; CLI and C ABI validation resolve the fetched tag to an immutable commit, read its checked-out manifest, and export that SHA to all build jobs, preventing tag movement from substituting unvalidated source. Real Git tag-movement regressions: `scripts/tests/test_release_security.py` | **FIXED** | -| TM-INF-028 | JS callback errors expose host stack traces | When a caller-supplied output or custom-builtin callback throws, propagating `error.stack` or an unsanitized message can expose host file paths and internal function names across the sandbox boundary | Native and wasm bindings propagate `error.message` only, strip absolute-path and `file://` segments, and cap diagnostics at 256 characters. Regressions: `onOutput error does not leak stack trace or host paths` in `crates/bashkit-js/__test__/streaming-output.spec.ts` and `callback errors do not leak host paths or stacks` in `crates/bashkit-wasm/__test__/bashkit-wasm.test.mjs`; mitigations annotated `THREAT[TM-INF-028]` in both binding implementations | **FIXED** | +| TM-INF-028 | JS callback errors expose host stack traces | When a caller-supplied output, custom-builtin or embedded-runtime host callback throws, propagating `error.stack` or an unsanitized message can expose host file paths and internal function names across the sandbox boundary | Native and wasm bindings propagate `error.message` only, strip absolute-path and `file://` segments, and cap diagnostics at 256 characters. Regressions: `onOutput error does not leak stack trace or host paths` in `crates/bashkit-js/__test__/streaming-output.spec.ts` and `callback errors do not leak host paths or stacks` in `crates/bashkit-wasm/__test__/bashkit-wasm.test.mjs`; mitigations annotated `THREAT[TM-INF-028]` in the binding implementations; embedded-runtime host callbacks also reject unsupported/deep JSON values and same-instance reentry | **FIXED** | | TM-INF-030 | Raw callback or custom-builtin errors leak host internals through ScriptedTool/ToolImpl/ToolRegistry | Tool callbacks and operator-provided ScriptedTool builtins may return errors containing API keys, connection strings, or stack traces; output is attacker-influenced and agent-visible | `sanitize_errors` defaults to true on `ScriptedTool`, `ToolImpl`, and the cross-runtime `ToolRegistry`; ScriptedTool wraps custom builtins so fatal errors use the same generic message while cancellation/resource-limit errors remain actionable; every shell/Python/TypeScript registry surface applies the boundary (`bashkit-scripted-tool`: `scripted_tool/execute.rs`, `scripted_tool/mod.rs`, `tool_def.rs`, `tool_registry.rs`) | **MITIGATED** | | TM-INF-031 | `final_env` capture bypasses output filtering and size caps | When `capture_final_env` is enabled, env contents are a user-visible output channel that could leak internal markers or exceed output limits | Visibility filtering + output-byte cap applied when building `final_env` (`interpreter/mod.rs`) | **MITIGATED** | | TM-INF-032 | Competitor fixture import executes untrusted upstream code in CI | Automatically downloading or mechanically copying third-party regression scripts into a host-shell oracle would let a compromised upstream run arbitrary commands with CI permissions; a source commit hash proves provenance, not trust | Competitor corpora are manually reviewed, data-only JSON checked into the repository with full commit provenance; CI performs no fetch; Bashkit cases run in the VFS with normal resource limits; only cases explicitly marked `real_bash` run under the host oracle, with a cleared environment and temporary cwd. The schema/classification gate is `competitor_corpus_has_complete_provenance_and_classification`; process contract is [Testing Strategy](../operations/testing.md) | **MITIGATED** | diff --git a/knowledge/status/capability-parity.md b/knowledge/status/capability-parity.md index 0e0becbc4..3902901f9 100644 --- a/knowledge/status/capability-parity.md +++ b/knowledge/status/capability-parity.md @@ -38,9 +38,11 @@ A dash means the feature is intentionally unsupported and has a recorded reason | Filesystem mounts beyond seeded virtual files | ✅ | ✅ | — | ✅ | ✅ | ✅ | — | — | | Snapshot and restore of interpreter state | ✅ | — | — | — | ✅ | ✅ | ✅ | — | | Explicit embedded Python opt-in | ✅ | ✅ | — | ✅ | ✅ | ✅ | — | — | -| Explicit embedded TypeScript opt-in | ✅ | ✅ | — | — | ✅ | — | — | — | +| Explicit embedded TypeScript opt-in | ✅ | ✅ | — | — | ✅ | ✅ | — | — | | Explicit embedded SQLite opt-in | ✅ | ✅ | — | ✅ | ✅ | ✅ | — | — | | In-process interactive terminal session (keystrokes in, screen text and per-command transcript out; vi, less) | ✅ | — | — | — | ✅ | ✅ | — | — | +| Named host callbacks invoked from embedded Python | ✅ | ✅ | — | — | ✅ | ✅ | — | — | +| Named host callbacks invoked from embedded TypeScript | ✅ | ✅ | — | — | ✅ | ✅ | — | — | ## Intentional exclusions @@ -74,6 +76,8 @@ A dash means the feature is intentionally unsupported and has a recorded reason - `runtime_typescript`: Language runtimes are intentionally absent from the logic-only shell. - `runtime_sqlite`: Language runtimes are intentionally absent from the logic-only shell. - `interactive_terminal`: ScriptedTool runs logic-only scripts per call and has no terminal device. +- `host_functions_python`: Language runtimes are intentionally absent from the logic-only shell. +- `host_functions_typescript`: Language runtimes are intentionally absent from the logic-only shell. ### CLI @@ -88,6 +92,8 @@ A dash means the feature is intentionally unsupported and has a recorded reason - `snapshots`: The CLI exposes no snapshot commands. - `runtime_typescript`: The CLI build does not register the TypeScript runtime. - `interactive_terminal`: The CLI's interactive mode runs on the host's real terminal (rustyline), not the in-process Terminal. +- `host_functions_python`: A standalone process cannot receive in-process host callbacks. +- `host_functions_typescript`: A standalone process cannot receive in-process host callbacks. ### Python @@ -100,7 +106,6 @@ A dash means the feature is intentionally unsupported and has a recorded reason - `stdin`: execute() accepts a command string and output callback, not top-level host stdin. - `host_call_suspension`: NAPI custom builtins await callbacks directly; no request and resume handle is exposed. - `transport_hooks`: NAPI exposes network policy and credentials but no JavaScript HttpTransport hook. -- `runtime_typescript`: The NAPI package does not compile or expose the TypeScript runtime. ### Browser WASM @@ -115,6 +120,8 @@ A dash means the feature is intentionally unsupported and has a recorded reason - `runtime_typescript`: The slim browser build omits embedded language runtimes. - `runtime_sqlite`: The slim browser build omits embedded language runtimes. - `interactive_terminal`: Not exposed in the slim browser package yet. +- `host_functions_python`: The slim browser build currently omits embedded language runtimes. +- `host_functions_typescript`: The slim browser build currently omits embedded language runtimes. ### C ABI @@ -132,6 +139,8 @@ A dash means the feature is intentionally unsupported and has a recorded reason - `runtime_typescript`: ABI v1 omits embedded language runtimes. - `runtime_sqlite`: ABI v1 omits embedded language runtimes. - `interactive_terminal`: Not exposed in the C ABI. +- `host_functions_python`: ABI v1 currently omits embedded language runtimes and runtime callback registration. +- `host_functions_typescript`: ABI v1 currently omits embedded language runtimes and runtime callback registration. ## Enforcement From bfa85048710291e240b5c2cf9d00338c5ff10164 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Fri, 16 Oct 2026 15:15:38 -0500 Subject: [PATCH 3/5] test(js): synchronize blocking access checks across platforms --- .../__test__/runtime-functions.spec.ts | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/crates/bashkit-js/__test__/runtime-functions.spec.ts b/crates/bashkit-js/__test__/runtime-functions.spec.ts index 047567b8d..878290d6a 100644 --- a/crates/bashkit-js/__test__/runtime-functions.spec.ts +++ b/crates/bashkit-js/__test__/runtime-functions.spec.ts @@ -165,24 +165,36 @@ for (const language of ["python", "typescript"] as const) { t.is(fs.readFile("/tmp/state"), "data"); }); - test(`${Factory.name}: ${language} sync access racing a new handler cannot deadlock`, async (t) => { - const bash = new Factory(options(() => 42)); - bash.writeFile("/tmp/state", "ready"); - let started!: () => void; + test(`${Factory.name}: ${language} sync access before a host call cannot deadlock`, async (t) => { + let started!: () => void, release!: () => void; const ready = new Promise((resolve) => { started = resolve; }); - const executing = bash.execute(`echo started; sleep 0.2; ${call}`, { - onOutput: () => { - started(); + const gate = new Promise((resolve) => { + release = resolve; + }); + const bash = new Factory({ + ...options(() => 42), + customBuiltins: { + wait_host: async () => { + started(); + await gate; + return ""; + }, }, }); + bash.writeFile("/tmp/state", "ready"); + const executing = bash.execute(`wait_host; ${call}`, { + onOutput: () => {}, + }); await ready; - await delay(0); + // Hold the shell before its runtime call. A blocking read would wait on + // the interpreter while the JS event loop is needed to release this gate. t.throws(() => bash.readFile("/tmp/state"), { message: /would block/ }); + release(); const result = await executing; t.is(result.exitCode, 0, result.stderr); - t.is(result.stdout, "started\n42\n"); + t.is(result.stdout, "42\n"); }); test(`${Factory.name}: ${language} sync invocation rejects host calls`, (t) => { @@ -275,7 +287,10 @@ for (const language of ["python", "typescript"] as const) { } test(`${language} host functions preserve HTTP request contexts and async I/O`, async (t) => { - const upstream = createServer((req, res) => res.end(req.url!.slice(1))); + const upstream = createServer((req, res) => { + res.setHeader("Content-Type", "text/plain; charset=utf-8"); + res.end(req.url!.slice(1)); + }); await new Promise((resolve) => upstream.listen(0, "127.0.0.1", resolve), ); @@ -291,6 +306,7 @@ for (const language of ["python", "typescript"] as const) { }), ); const server = createServer((req, res) => { + res.setHeader("Content-Type", "text/plain; charset=utf-8"); void request.run(req.url!.slice(1), async () => { try { res.end((await bash.execute(call)).stdout.trim()); From 0efb530c4a1d27c43057253c49a544d66c20d0c6 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Fri, 16 Oct 2026 15:56:54 -0500 Subject: [PATCH 4/5] fix(js): safely release private runtimes during worker teardown --- .../fixtures/runtime-functions-shutdown.mjs | 98 +++++++++++++++++++ .../__test__/runtime-functions.spec.ts | 16 +++ crates/bashkit-js/src/lib.rs | 40 ++++++-- docs/start-node.md | 5 +- 4 files changed, 151 insertions(+), 8 deletions(-) create mode 100644 crates/bashkit-js/__test__/fixtures/runtime-functions-shutdown.mjs diff --git a/crates/bashkit-js/__test__/fixtures/runtime-functions-shutdown.mjs b/crates/bashkit-js/__test__/fixtures/runtime-functions-shutdown.mjs new file mode 100644 index 000000000..b5fdfbc4e --- /dev/null +++ b/crates/bashkit-js/__test__/fixtures/runtime-functions-shutdown.mjs @@ -0,0 +1,98 @@ +// Run in a subprocess: native runtime panics must not disappear into AVA's +// worker stderr. Host functions defer invocation beyond the NAPI stack frame. +import assert from "node:assert/strict"; +import { + isMainThread, + parentPort, + Worker, + workerData, +} from "node:worker_threads"; +import { Bash, BashTool } from "../../wrapper.js"; +import { ScriptedTool } from "../../scripted.js"; + +if (isMainThread) { + for (const kind of ["bash", "tool"]) { + for (const language of ["python", "typescript"]) { + for (const style of ["pending", "busy"]) { + const worker = new Worker(new URL(import.meta.url), { + workerData: { kind, language, style }, + }); + await new Promise((resolve, reject) => { + const timer = setTimeout( + () => + reject( + new Error( + `host handler did not start: ${kind}/${language}/${style}`, + ), + ), + 15000, + ); + worker.once("message", () => { + clearTimeout(timer); + resolve(); + }); + worker.once("error", (error) => { + clearTimeout(timer); + reject(error); + }); + }); + await worker.terminate(); + } + } + } + // ScriptedTool has a synchronous callback ABI; verify its ordinary completed + // worker teardown, without interrupting V8 inside that native callback frame. + const worker = new Worker(new URL(import.meta.url), { + workerData: { kind: "scripted" }, + }); + await new Promise((resolve, reject) => { + let seen = false; + worker.once("message", (stdout) => { + assert.equal(stdout.trim(), "42"); + seen = true; + }); + worker.once("error", reject); + worker.once("exit", (code) => + code === 0 && seen + ? resolve() + : reject(new Error(`scripted worker exited ${code}`)), + ); + }); + assert.equal( + (await new Bash().execute("echo reusable")).stdout, + "reusable\n", + ); + console.log("pending worker shutdown passed"); +} else if (workerData.kind === "scripted") { + const tool = new ScriptedTool({ name: "shutdown" }); + tool.addTool("lookup", "Return a value", () => "42"); + void tool + .execute("lookup") + .then((result) => parentPort.postMessage(result.stdout)); +} else { + const handler = () => { + parentPort.postMessage("started"); + if (workerData.style === "busy") { + while (true) { + /* host JS interrupted by Worker.terminate */ + } + } + return new Promise(() => {}); + }; + const Factory = workerData.kind === "bash" ? Bash : BashTool; + const python = workerData.language === "python"; + const bash = new Factory( + python + ? { + python: true, + externalFunctions: ["lookup"], + externalHandler: handler, + } + : { + typescript: true, + typescriptExternalFunctions: ["lookup"], + typescriptExternalHandler: handler, + }, + ); + void bash.execute(python ? "python -c 'lookup()'" : "ts -c 'await lookup()'"); +} diff --git a/crates/bashkit-js/__test__/runtime-functions.spec.ts b/crates/bashkit-js/__test__/runtime-functions.spec.ts index 878290d6a..58f1e7ebf 100644 --- a/crates/bashkit-js/__test__/runtime-functions.spec.ts +++ b/crates/bashkit-js/__test__/runtime-functions.spec.ts @@ -4,6 +4,9 @@ import test from "ava"; import { AsyncLocalStorage } from "node:async_hooks"; import { Worker } from "node:worker_threads"; import { createServer } from "node:http"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; import { Bash, BashTool } from "../wrapper.js"; import { createBashTool } from "../langchain.js"; import { bashTool as aiBashTool } from "../ai.js"; @@ -526,3 +529,16 @@ test("both runtimes dispatch independently and advertise host functions", async t.regex(metadata, /TypeScript host functions: lookup/); } }); + +test("pending worker shutdown releases native runtimes without panics", async (t) => { + const fixture = fileURLToPath( + new URL("./fixtures/runtime-functions-shutdown.mjs", import.meta.url), + ); + const { stdout, stderr } = await promisify(execFile)( + process.execPath, + [fixture], + { timeout: 30000 }, + ); + t.is(stdout.trim(), "pending worker shutdown passed"); + t.is(stderr, ""); +}); diff --git a/crates/bashkit-js/src/lib.rs b/crates/bashkit-js/src/lib.rs index 74939949e..c79de2991 100644 --- a/crates/bashkit-js/src/lib.rs +++ b/crates/bashkit-js/src/lib.rs @@ -94,6 +94,34 @@ const SYNC_BUILTIN_DEADLOCK_ERROR: &str = "custom builtins require execute() (as // callback so untrusted scripts cannot hang the process indefinitely. const SYNC_SCRIPTED_TOOL_DEADLOCK_ERROR: &str = "registered tools require execute() (async). executeSync() would deadlock because the JS event loop is blocked while the synchronous call is in flight"; +// A worker may disappear while the last native owner is held by an async +// task. Tokio forbids blocking Runtime::drop in that context; shut it down +// without waiting there, while preserving graceful drop on ordinary threads. +struct OwnedRuntime(Option); + +impl OwnedRuntime { + fn new(runtime: tokio::runtime::Runtime) -> Self { + Self(Some(runtime)) + } + + fn block_on(&self, future: F) -> F::Output { + self.0 + .as_ref() + .expect("runtime is live until drop") + .block_on(future) + } +} + +impl Drop for OwnedRuntime { + fn drop(&mut self) { + if tokio::runtime::Handle::try_current().is_ok() + && let Some(runtime) = self.0.take() + { + runtime.shutdown_background(); + } + } +} + struct OnOutputReentryScope { depth: Arc, } @@ -1463,7 +1491,7 @@ fn require_snapshot_hmac_key(options: Option<&SnapshotOptions>) -> napi::Result< struct SharedState { inner: Mutex, - rt: Mutex, + rt: Mutex, cancelled: std::sync::Mutex>, on_output_reentry_depth: Arc, /// Tracks whether the JS event loop is currently blocked inside an @@ -3398,7 +3426,7 @@ pub struct ScriptedTool { short_desc: Option, tools: Vec, env_vars: Vec<(String, String)>, - rt: Mutex, + rt: Mutex, max_commands: Option, max_loop_iterations: Option, in_sync_execute_depth: Arc, @@ -3418,7 +3446,7 @@ impl ScriptedTool { short_desc: options.short_description, tools: Vec::new(), env_vars: Vec::new(), - rt: Mutex::new(rt), + rt: Mutex::new(OwnedRuntime::new(rt)), max_commands: options.max_commands, max_loop_iterations: options.max_loop_iterations, in_sync_execute_depth: Arc::new(AtomicUsize::new(0)), @@ -3922,12 +3950,12 @@ fn shared_state_from_opts(opts: BashOptions) -> napi::Result { // Build a temporary SharedState to pass to build_bash_from_state let tmp = SharedState { inner: Mutex::new(RustBash::new()), - rt: Mutex::new( + rt: Mutex::new(OwnedRuntime::new( tokio::runtime::Builder::new_current_thread() .enable_all() .build() .map_err(|e| napi::Error::from_reason(format!("Failed to create runtime: {e}")))?, - ), + )), cancelled: std::sync::Mutex::new(Arc::new(AtomicBool::new(false))), on_output_reentry_depth: Arc::new(AtomicUsize::new(0)), in_sync_execute_depth: Arc::new(AtomicUsize::new(0)), @@ -3986,7 +4014,7 @@ fn shared_state_from_opts(opts: BashOptions) -> napi::Result { Ok(SharedState { inner: Mutex::new(bash), - rt: Mutex::new(rt), + rt: Mutex::new(OwnedRuntime::new(rt)), cancelled: std::sync::Mutex::new(cancelled), on_output_reentry_depth: tmp.on_output_reentry_depth, in_sync_execute_depth: tmp.in_sync_execute_depth, diff --git a/docs/start-node.md b/docs/start-node.md index bd148f008..aaff168fc 100644 --- a/docs/start-node.md +++ b/docs/start-node.md @@ -31,7 +31,7 @@ const bash = new Bash({ python: true, externalFunctions: ["lookup"], externalHandler: async (name, args, kwargs, { signal }) => { - const response = await fetch(`https://api.example.com/items/${args[0]}`, { signal }); + const response = await fetch(`https://api.example.com/items/${encodeURIComponent(String(args[0]))}`, { signal }); return response.json(); }, typescript: true, @@ -58,7 +58,8 @@ Its handler signal aborts on cancellation, timeout or completion; pass it to hos I/O. A callback cannot reenter its own interpreter or live filesystem. Use another instance for nested execution. Independent asynchronous requests can share an instance and queue safely. Synchronous access during asynchronous execution fails -promptly. Callbacks survive reset; supply the options again when restoring a +promptly. Independent workers may own instances, including workers stopped with +a pending host function. Callbacks survive reset; supply the options again when restoring a snapshot, since host code and request context are never serialized. Host functions run trusted application code. Enforce service authorization and From c14526757bd24aa0d9f97bc3f7974185a4393ded Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Fri, 16 Oct 2026 15:57:00 -0500 Subject: [PATCH 5/5] chore(knowledge): record native worker shutdown ownership --- knowledge/log.md | 2 +- knowledge/runtimes/node-package.md | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/knowledge/log.md b/knowledge/log.md index 68d565981..85e7169d6 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,7 +2,7 @@ ## 2026-10-16 -* **Node runtime parity**: Added Python and TypeScript named host callbacks to Bash/BashTool and JS agent adapters, with per-execution caller context, cooperative abort, reentry guards and worker/platform tests. Added explicit runtime host-function capability rows and a shared bounded JSON codec for foreign bindings. See [Node Package](runtimes/node-package.md). +* **Node runtime parity**: Added Python and TypeScript named host callbacks to Bash/BashTool and JS agent adapters, with per-execution caller context, cooperative abort, reentry guards and worker/platform tests. Private runtime ownership also handles forced worker teardown without dropping a Tokio runtime inside an async task. Added explicit runtime host-function capability rows and a shared bounded JSON codec for foreign bindings. See [Node Package](runtimes/node-package.md). * **Python / agent integrations**: Added Python and TypeScript host function callbacks to `BashTool`, LangChain and deepagents. Reuse per-execution callback sessions for caller-loop affinity, ContextVars and cancellation; sync execution uses the dedicated private worker loop. Reentry is callback-context-specific so independent requests queue safely. Private-loop timeout cancellation now targets its own work item; native result deliveries drain before interpreter finalization to prevent a shutdown race. Added FastAPI, real Jupyter, alternate-loop and cross-thread regression coverage plus macOS/Windows callback CI. See [Python Package](runtimes/python-package.md). diff --git a/knowledge/runtimes/node-package.md b/knowledge/runtimes/node-package.md index 6d9545fcc..2137a923a 100644 --- a/knowledge/runtimes/node-package.md +++ b/knowledge/runtimes/node-package.md @@ -36,6 +36,14 @@ access fails during asynchronous execution; asynchronous entry waits for any earlier synchronous entry to finish. Sync host calls fail explicitly; ordinary synchronous commands remain available. Separate worker-thread environments own separate dispatchers; weak references do not keep Node alive at shutdown. +If worker termination drops the last private-runtime owner inside an async task, +the owner uses Tokio's nonblocking shutdown instead of Runtime::drop (which +panics in that context). Ordinary thread teardown keeps graceful runtime drop. +A subprocess regression captures native stderr while terminating workers during +pending and CPU-bound Python/TypeScript host callbacks; completed ScriptedTool +worker teardown is covered too. ScriptedTool retains its synchronous callback +ABI; force-interrupting V8 inside such a native callback frame is outside this +async host-function shutdown contract. `runtime-functions.spec.ts` exercises real HTTP request contexts, aborts, timeout, queued calls, worker threads, adapters, reentry, bad values and snapshot/reset.