diff --git a/.github/workflows/decision-trajectories.yml b/.github/workflows/decision-trajectories.yml index a1a3e5e..ab2223d 100644 --- a/.github/workflows/decision-trajectories.yml +++ b/.github/workflows/decision-trajectories.yml @@ -3,9 +3,9 @@ name: Private decision trajectories on: push: branches: [master] - paths: [src/bitworld/native_websocket.nim, tests/support/native_websocket_probe.nim, tests/test_native_websocket.py, src/bitworld/decision_trajectory.nim, src/bitworld/runtime.nim, src/bitworld/native_http.nim, src/bitworld/native_stop.nim, src/bitworld/artifact_runtime.nim, tests/test_artifact_http.py, tests/support/artifact_http_probe.nim, tests/test_native_https.py, tests/support/native_https_probe.nim, tests/test_decision_trajectory.nim, tests/test_native_http.py, tests/support/native_http_probe.nim, tests/support/native_request_control_probe.nim, tests/test_native_request_control.py, .github/workflows/decision-trajectories.yml] + paths: [src/bitworld/runtime_input.nim, tests/test_runtime.nim, tests/support/runtime_input_probe.nim, tests/support/runtime_config_input_probe.nim, tests/test_runtime_input.py, tests/test_runtime_input_https.py, tests/test_runtime_config_input.py, src/bitworld/native_websocket.nim, tests/support/native_websocket_probe.nim, tests/test_native_websocket.py, src/bitworld/decision_trajectory.nim, src/bitworld/runtime.nim, src/bitworld/native_http.nim, src/bitworld/native_stop.nim, src/bitworld/artifact_runtime.nim, tests/test_artifact_http.py, tests/support/artifact_http_probe.nim, tests/test_native_https.py, tests/support/native_https_probe.nim, tests/test_decision_trajectory.nim, tests/test_native_http.py, tests/support/native_http_probe.nim, tests/support/native_request_control_probe.nim, tests/test_native_request_control.py, .github/workflows/decision-trajectories.yml] pull_request: - paths: [src/bitworld/native_websocket.nim, tests/support/native_websocket_probe.nim, tests/test_native_websocket.py, src/bitworld/decision_trajectory.nim, src/bitworld/runtime.nim, src/bitworld/native_http.nim, src/bitworld/native_stop.nim, src/bitworld/artifact_runtime.nim, tests/test_artifact_http.py, tests/support/artifact_http_probe.nim, tests/test_native_https.py, tests/support/native_https_probe.nim, tests/test_decision_trajectory.nim, tests/test_native_http.py, tests/support/native_http_probe.nim, tests/support/native_request_control_probe.nim, tests/test_native_request_control.py, .github/workflows/decision-trajectories.yml] + paths: [src/bitworld/runtime_input.nim, tests/test_runtime.nim, tests/support/runtime_input_probe.nim, tests/support/runtime_config_input_probe.nim, tests/test_runtime_input.py, tests/test_runtime_input_https.py, tests/test_runtime_config_input.py, src/bitworld/native_websocket.nim, tests/support/native_websocket_probe.nim, tests/test_native_websocket.py, src/bitworld/decision_trajectory.nim, src/bitworld/runtime.nim, src/bitworld/native_http.nim, src/bitworld/native_stop.nim, src/bitworld/artifact_runtime.nim, tests/test_artifact_http.py, tests/support/artifact_http_probe.nim, tests/test_native_https.py, tests/support/native_https_probe.nim, tests/test_decision_trajectory.nim, tests/test_native_http.py, tests/support/native_http_probe.nim, tests/support/native_request_control_probe.nim, tests/test_native_request_control.py, .github/workflows/decision-trajectories.yml] jobs: test: @@ -53,3 +53,11 @@ jobs: run: | nim c -d:release --threads:on --mm:orc --path:src -o:/tmp/native-websocket-probe tests/support/native_websocket_probe.nim python3 tests/test_native_websocket.py /tmp/native-websocket-probe + - name: Verify bounded startup inputs and private capture before validation + run: | + nim c -r --parallelBuild:1 --threads:on --mm:orc --path:src -o:/tmp/test-runtime tests/test_runtime.nim + nim c --parallelBuild:1 -d:release --threads:on --mm:orc --path:src -o:/tmp/runtime-input-probe tests/support/runtime_input_probe.nim + python3 tests/test_runtime_input.py /tmp/runtime-input-probe + python3 tests/test_runtime_input_https.py /tmp/runtime-input-probe + nim c --parallelBuild:1 -d:release --threads:on --mm:orc --path:src -o:/tmp/runtime-config-input-probe tests/support/runtime_config_input_probe.nim + python3 tests/test_runtime_config_input.py /tmp/runtime-config-input-probe diff --git a/README.md b/README.md index 601dc8c..abd4b0a 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,26 @@ This makes the game world useful as a sandbox for questions like: - What incentives cause betrayal? - How do agents adapt to repeated social interaction? +## Coworld startup inputs + +`readRuntimeConfig`, `readCogameUri`, and `readCogameEnv` require an +`InputReader`. Native games supply a closure calling +`runtime_input.readRuntimeInput` with one absolute startup deadline, an owned +`NativeRequestControl`, body/header byte limits, and private captures. Reserve +the game's cleanup budget before choosing that deadline. + +The reader owns each HTTP(S) handle until it joins. It retains received bytes +before status, UTF-8, or game configuration validation. Redirects are rejected; +TLS certificate and hostname verification remain enabled. `SSL_CERT_FILE` +selects an explicit process trust bundle. File inputs must be regular files +and obey the same byte limit and deadline. + +`runtimeInputCapturesJson` contains private source URIs and raw bytes. Keep it +in the private checkpoint, never public replay or process logs. Games seal +failed or interrupted initialization only after input ownership ends, using +the original bounded cleanup deadline. `pathFromCogameUri` decodes local file +URIs; it never downloads inputs. + ## Visual Style Bit World is designed around strict retro display constraints: diff --git a/src/bitworld/native_http.nim b/src/bitworld/native_http.nim index 3e4e782..9c1ff9e 100644 --- a/src/bitworld/native_http.nim +++ b/src/bitworld/native_http.nim @@ -13,9 +13,9 @@ type ArtifactHttpMethod* = enum ahPut = "PUT", ahPost = "POST" RequestPurpose = enum - rpInference, rpArtifact + rpInference, rpInput, rpArtifact NativeHttpKind* = enum - nhComplete, nhDeadline, nhInterrupted, nhCanceled, nhTransportFailure + nhComplete, nhDeadline, nhInterrupted, nhCanceled, nhTransportFailure, nhLimitExceeded NativeRequestControl* = object canceled: Atomic[bool] NativeHttpResponse* = object @@ -31,6 +31,8 @@ type purpose: RequestPurpose control: ptr NativeRequestControl headerBytes, bodyBytes: string + maxHeaderBytes, maxBodyBytes: int + limitExceeded: bool # The pinned Nim binding omits these existing libcurl options. const @@ -59,6 +61,12 @@ proc receiveHeaders(buffer: cstring, size, count: int, context: pointer): int {. let transfer = cast[ptr Transfer](context) result = size * count let offset = transfer.headerBytes.len + if result > transfer.maxHeaderBytes - offset: + let retained = transfer.maxHeaderBytes - offset + transfer.headerBytes.setLen(offset + retained) + if retained > 0: copyMem(transfer.headerBytes[offset].addr, buffer, retained) + transfer.limitExceeded = true + return 0 transfer.headerBytes.setLen(offset + result) if result > 0: copyMem(transfer.headerBytes[offset].addr, buffer, result) @@ -66,23 +74,30 @@ proc receiveBody(buffer: cstring, size, count: int, context: pointer): int {.cde let transfer = cast[ptr Transfer](context) result = size * count let offset = transfer.bodyBytes.len + if result > transfer.maxBodyBytes - offset: + let retained = transfer.maxBodyBytes - offset + transfer.bodyBytes.setLen(offset + retained) + if retained > 0: copyMem(transfer.bodyBytes[offset].addr, buffer, retained) + transfer.limitExceeded = true + return 0 transfer.bodyBytes.setLen(offset + result) if result > 0: copyMem(transfer.bodyBytes[offset].addr, buffer, result) proc checkTransfer(context: pointer, downloadTotal, downloaded, uploadTotal, uploaded: int64): cint {.cdecl.} = let transfer = cast[ptr Transfer](context) - if (transfer.purpose == rpInference and + if (transfer.purpose != rpArtifact and (interruptionRequested() or transfer.control[].nativeRequestCanceled())) or getMonoTime() >= transfer.deadline: 1 else: 0 -proc performOwnedRequest(url: string, httpMethod: ArtifactHttpMethod, +proc performOwnedRequest(url: string, httpMethod: string, headers: HttpHeaders, body: string, deadline: MonoTime, - purpose: RequestPurpose, control: var NativeRequestControl): NativeHttpResponse = - if purpose == rpInference and interruptionRequested(): + purpose: RequestPurpose, control: var NativeRequestControl, + maxBodyBytes, maxHeaderBytes: int): NativeHttpResponse = + if purpose != rpArtifact and interruptionRequested(): result.kind = nhInterrupted return - if purpose == rpInference and control.nativeRequestCanceled(): + if purpose != rpArtifact and control.nativeRequestCanceled(): result.kind = nhCanceled return let remaining = (deadline - getMonoTime()).inNanoseconds @@ -93,7 +108,8 @@ proc performOwnedRequest(url: string, httpMethod: ArtifactHttpMethod, let handle = easy_init() doAssert handle != nil, "Cannot allocate native HTTP handle" var headerList: Pslist - var transfer = Transfer(deadline: deadline, purpose: purpose, control: control.addr) + var transfer = Transfer(deadline: deadline, purpose: purpose, control: control.addr, + maxBodyBytes: maxBodyBytes, maxHeaderBytes: maxHeaderBytes) var oldMask, pipeMask, previousPending: Sigset doAssert sigemptyset(pipeMask) == 0 doAssert sigaddset(pipeMask, SIGPIPE) == 0 @@ -107,9 +123,10 @@ proc performOwnedRequest(url: string, httpMethod: ArtifactHttpMethod, doAssert appended != nil, "Cannot allocate native HTTP headers" headerList = appended requireCurl(handle.easy_setopt(OPT_URL, url.cstring)) - requireCurl(handle.easy_setopt(OPT_CUSTOMREQUEST, ($httpMethod).cstring)) - requireCurl(handle.easy_setopt(OPT_POSTFIELDS, body.cstring)) - requireCurl(handle.easy_setopt(OPT_POSTFIELDSIZE, clong(body.len))) + requireCurl(handle.easy_setopt(OPT_CUSTOMREQUEST, httpMethod.cstring)) + if httpMethod != "GET": + requireCurl(handle.easy_setopt(OPT_POSTFIELDS, body.cstring)) + requireCurl(handle.easy_setopt(OPT_POSTFIELDSIZE, clong(body.len))) requireCurl(handle.easy_setopt(OPT_HTTPHEADER, headerList)) requireCurl(handle.easy_setopt(OPT_FOLLOWLOCATION, clong(0))) requireCurl(handle.easy_setopt(OptProtocols, clong(3))) # HTTP and HTTPS only. @@ -125,10 +142,10 @@ proc performOwnedRequest(url: string, httpMethod: ArtifactHttpMethod, requireCurl(handle.easy_setopt(OptXferInfoFunction, checkTransfer)) let started = getMonoTime() let finalRemaining = (deadline - started).inNanoseconds - if finalRemaining <= 0 or (purpose == rpInference and + if finalRemaining <= 0 or (purpose != rpArtifact and (interruptionRequested() or control.nativeRequestCanceled())): - result.kind = if purpose == rpInference and interruptionRequested(): nhInterrupted - elif purpose == rpInference and control.nativeRequestCanceled(): nhCanceled + result.kind = if purpose != rpArtifact and interruptionRequested(): nhInterrupted + elif purpose != rpArtifact and control.nativeRequestCanceled(): nhCanceled else: nhDeadline return let milliseconds = clong((finalRemaining + 999_999) div 1_000_000) @@ -140,8 +157,9 @@ proc performOwnedRequest(url: string, httpMethod: ArtifactHttpMethod, var status: clong requireCurl(handle.easy_getinfo(INFO_RESPONSE_CODE, status.addr)) if status != 0: result.httpStatus = some(int(status)) - if purpose == rpInference and interruptionRequested(): result.kind = nhInterrupted - elif purpose == rpInference and control.nativeRequestCanceled(): result.kind = nhCanceled + if purpose != rpArtifact and interruptionRequested(): result.kind = nhInterrupted + elif purpose != rpArtifact and control.nativeRequestCanceled(): result.kind = nhCanceled + elif transfer.limitExceeded: result.kind = nhLimitExceeded elif code == E_OPERATION_TIMEOUTED or getMonoTime() >= deadline: result.kind = nhDeadline elif code == E_OK: result.kind = nhComplete else: result.kind = nhTransportFailure @@ -157,8 +175,8 @@ proc performOwnedRequest(url: string, httpMethod: ArtifactHttpMethod, doAssert sigwait(pipeMask, received) == 0 var discardedMask: Sigset doAssert pthread_sigmask(SIG_SETMASK, oldMask, discardedMask) == 0 - if purpose == rpInference and interruptionRequested(): result.kind = nhInterrupted - elif purpose == rpInference and control.nativeRequestCanceled(): result.kind = nhCanceled + if purpose != rpArtifact and interruptionRequested(): result.kind = nhInterrupted + elif purpose != rpArtifact and control.nativeRequestCanceled(): result.kind = nhCanceled elif getMonoTime() >= deadline: result.kind = nhDeadline result.headerBytes = move transfer.headerBytes result.bodyBytes = move transfer.bodyBytes @@ -167,11 +185,20 @@ proc performOwnedRequest(url: string, httpMethod: ArtifactHttpMethod, proc performNativePost*(url: string, headers: HttpHeaders, body: string, deadline: MonoTime, control: var NativeRequestControl): NativeHttpResponse = ## A caller shares one deadline across retries. Never reset it per attempt. - performOwnedRequest(url, ahPost, headers, body, deadline, rpInference, control) + performOwnedRequest(url, "POST", headers, body, deadline, rpInference, control, + int.high, int.high) proc performArtifactUpload*(url: string, httpMethod: ArtifactHttpMethod, headers: HttpHeaders, body: string, cleanupDeadline: MonoTime): NativeHttpResponse = ## Checkpoint finalization has its own finite cleanup lifetime after inference stops. ## No caller can disable interruption in the inference API. var control: NativeRequestControl - performOwnedRequest(url, httpMethod, headers, body, cleanupDeadline, rpArtifact, control) + performOwnedRequest(url, $httpMethod, headers, body, cleanupDeadline, rpArtifact, control, + int.high, int.high) + +proc performInputGet*(url: string, headers: HttpHeaders, deadline: MonoTime, + control: var NativeRequestControl, maxBodyBytes, maxHeaderBytes: int): NativeHttpResponse = + ## Startup input has one owner/deadline and never follows a redirect to a new URI. + doAssert maxBodyBytes > 0 and maxHeaderBytes > 0 + performOwnedRequest(url, "GET", headers, "", deadline, rpInput, control, + maxBodyBytes, maxHeaderBytes) diff --git a/src/bitworld/runtime.nim b/src/bitworld/runtime.nim index e920545..e564d1e 100644 --- a/src/bitworld/runtime.nim +++ b/src/bitworld/runtime.nim @@ -1,5 +1,5 @@ import - std/[os, parseopt, strutils], + std/[os, parseopt, strutils, unicode], curly const @@ -14,6 +14,8 @@ const CogamePortEnv* = "COGAME_PORT" type + InputReader* = proc(value, source: string): string {.closure.} + CogameRuntimeError* = object of CatchableError RuntimeConfig* = object @@ -94,40 +96,16 @@ proc isHttpCogameUri*(value: string): bool = ## Returns true when a Coworld URI is an HTTP(S) URI. value.startsWith("http://") or value.startsWith("https://") -proc readCogameUri*(value, source: string): string = - ## Reads data from a Coworld file URI or HTTP(S) signed URI. - if value.len == 0: - return "" +proc readCogameUri*(value, source: string, inputReader: InputReader): string = + ## The caller owns input deadlines, cancellation, limits, and private capture. + if value.len == 0: return "" + inputReader(value, source) - let path = filePathFromCogameUri(value, source) - if path.len > 0: - return readFile(path) - - if value.isHttpCogameUri(): - let client = newCurlPool(1) - defer: client.close() - let response = client.get(value) - if response.code < 200 or response.code >= 300: - raise newException( - IOError, - source & " download failed: " & $response.code - ) - return response.body - - if "://" in value: - raise newException( - CogameRuntimeError, - "unsupported URI from " & source & ": " & value - ) - - raise newException(CogameRuntimeError, source & " must be a URI") - -proc readCogameEnv*(name: string): string = - ## Reads data from a Coworld URI environment variable. - readCogameUri(getEnv(name), name) +proc readCogameEnv*(name: string, inputReader: InputReader): string = + readCogameUri(getEnv(name), name, inputReader) proc pathFromCogameUri*(value, source: string): string = - ## Converts a Coworld file/input URI into a local path. + ## Converts a Coworld local file URI into a path without performing transport. if value.len == 0: return "" @@ -135,11 +113,6 @@ proc pathFromCogameUri*(value, source: string): string = if path.len > 0: return path - if value.isHttpCogameUri(): - result = getTempDir() / ("cogame-" & source.toLowerAscii()) - writeFile(result, readCogameUri(value, source)) - return - if "://" in value: raise newException( CogameRuntimeError, @@ -276,7 +249,7 @@ proc writeRuntimeTarget( return value.writeLocalTarget(data) -proc readRuntimeConfig*(): RuntimeConfig = +proc readRuntimeConfig*(inputReader: InputReader): RuntimeConfig = ## Reads the Coworld runtime config from CLI arguments and env vars. result = RuntimeConfig(host: RuntimeDefaultHost, port: RuntimeDefaultPort) var @@ -308,11 +281,11 @@ proc readRuntimeConfig*(): RuntimeConfig = configSet = true of "config-path": key.requireValue(val) - result.config = readFile(val) + result.config = inputReader("file://" & absolutePath(val), "--" & key) configSet = true of "config-uri": key.requireValue(val) - result.config = readCogameUri(val, "--" & key) + result.config = readCogameUri(val, "--" & key, inputReader) configSet = true of "results": key.requireValue(val) @@ -332,12 +305,12 @@ proc readRuntimeConfig*(): RuntimeConfig = saveReplaySet = true of "load-replay": key.requireValue(val) - result.replay = readFile(val) + result.replay = inputReader("file://" & absolutePath(val), "--" & key) result.replayMode = true loadReplaySet = true of "load-replay-uri": key.requireValue(val) - result.replay = readCogameUri(val, "--" & key) + result.replay = readCogameUri(val, "--" & key, inputReader) result.replayMode = true loadReplaySet = true of "log": @@ -383,7 +356,7 @@ proc readRuntimeConfig*(): RuntimeConfig = if not configSet: let configUri = getEnv(CogameConfigUriEnv) if configUri.len > 0: - result.config = readCogameUri(configUri, CogameConfigUriEnv) + result.config = readCogameUri(configUri, CogameConfigUriEnv, inputReader) if not resultsSet: result.resultsUri = getEnv(CogameResultsUriEnv) if not saveReplaySet: @@ -391,10 +364,12 @@ proc readRuntimeConfig*(): RuntimeConfig = if not loadReplaySet: let replayUri = getEnv(CogameLoadReplayUriEnv) if replayUri.len > 0: - result.replay = readCogameUri(replayUri, CogameLoadReplayUriEnv) + result.replay = readCogameUri(replayUri, CogameLoadReplayUriEnv, inputReader) result.replayMode = true if not logSet: result.logUri = getEnv(CogameLogUriEnv) + if result.config.validateUtf8() != -1: + raise newException(CogameRuntimeError, "runtime config must be UTF-8") proc writeResults*(config: RuntimeConfig, data: string) = ## Writes a Coworld results artifact if a target is configured. diff --git a/src/bitworld/runtime_input.nim b/src/bitworld/runtime_input.nim new file mode 100644 index 0000000..250435a --- /dev/null +++ b/src/bitworld/runtime_input.nim @@ -0,0 +1,66 @@ +## Native startup input ownership. Keep transport out of runtime/WASM decoders. +import std/[base64, json, monotimes, options, os] +import native_http, native_stop, runtime + +type RuntimeInputCapture* = object + source*, uri*: string + response*: Option[NativeHttpResponse] + fileBytes*: string + +proc readRuntimeInput*(value, source: string, deadline: MonoTime, + control: var NativeRequestControl, maxBodyBytes, maxHeaderBytes: int, + captures: var seq[RuntimeInputCapture]): string = + ## Capture precedes status/format validation; the HTTP owner has already joined. + doAssert maxBodyBytes > 0 and maxBodyBytes < int.high and maxHeaderBytes > 0 + if interruptionRequested(): + raise newException(CogameRuntimeError, source & " input interrupted") + if control.nativeRequestCanceled(): + raise newException(CogameRuntimeError, source & " input canceled") + if getMonoTime() >= deadline: + raise newException(CogameRuntimeError, source & " input deadline exceeded") + if value.isHttpCogameUri(): + let response = performInputGet(value, @[], deadline, control, + maxBodyBytes, maxHeaderBytes) + captures.add RuntimeInputCapture(source: source, uri: value, response: some(response)) + if response.kind != nhComplete: + raise newException(CogameRuntimeError, source & " input " & $response.kind) + let status = response.httpStatus.get() + if status < 200 or status >= 300: + raise newException(CogameRuntimeError, source & " input HTTP status " & $status) + result = response.bodyBytes + else: + let path = pathFromCogameUri(value, source) + if getFileInfo(path).kind != pcFile: + raise newException(CogameRuntimeError, source & " input must be a regular file") + let file = open(path, fmRead) + try: + result = newString(maxBodyBytes + 1) + result.setLen(file.readBuffer(result[0].addr, result.len)) + finally: + file.close() + captures.add RuntimeInputCapture(source: source, uri: value, fileBytes: result) + if result.len > maxBodyBytes: + raise newException(CogameRuntimeError, source & " input byte limit exceeded") + if interruptionRequested(): + raise newException(CogameRuntimeError, source & " input interrupted") + if control.nativeRequestCanceled(): + raise newException(CogameRuntimeError, source & " input canceled") + if getMonoTime() >= deadline: + raise newException(CogameRuntimeError, source & " input deadline exceeded") + +proc runtimeInputCapturesJson*(captures: openArray[RuntimeInputCapture]): JsonNode = + ## Private evidence only: source URIs and bytes must never enter public replay/logs. + result = newJArray() + for capture in captures: + var item = %*{"source": capture.source, "uri": capture.uri, + "file_body_b64": encode(capture.fileBytes), "transport": newJNull()} + if capture.response.isSome: + let response = capture.response.get() + item["file_body_b64"] = newJNull() + item["transport"] = %*{"kind": $response.kind, + "http_status": response.httpStatus, "response_body_b64": encode(response.bodyBytes), + "response_headers_b64": encode(response.headerBytes), + "response_complete": response.transferComplete, + "response_reader_joined": response.responseReaderJoined, + "latency_ms": response.latencyMs, "error": response.error} + result.add item diff --git a/tests/support/artifact_http_probe.nim b/tests/support/artifact_http_probe.nim index 872f810..2b8c350 100644 --- a/tests/support/artifact_http_probe.nim +++ b/tests/support/artifact_http_probe.nim @@ -1,5 +1,5 @@ import std/[base64, json, monotimes, options, os, strutils, times] -import bitworld/[artifact_runtime, decision_trajectory, native_http, native_stop, runtime] +import bitworld/[artifact_runtime, decision_trajectory, native_http, native_stop, runtime, runtime_input] var control: NativeRequestControl let args = commandLineParams() @@ -15,7 +15,9 @@ if args[1] == "retained": for entry in walkDir("/proc/self/task"): inc initialThreads for index in 0 ..< 8: writeCogameUri(args[0], "retained", "application/json", "fixture") - doAssert readCogameUri(args[0], "fixture") == "reloaded" + var captures: seq[RuntimeInputCapture] + doAssert readRuntimeInput(args[0], "fixture", deadline, control, + 1024, 4096, captures) == "reloaded" var finalThreads = 0 for entry in walkDir("/proc/self/task"): inc finalThreads doAssert finalThreads == initialThreads diff --git a/tests/support/runtime_config_input_probe.nim b/tests/support/runtime_config_input_probe.nim new file mode 100644 index 0000000..aaa9cdc --- /dev/null +++ b/tests/support/runtime_config_input_probe.nim @@ -0,0 +1,16 @@ +import std/[json, monotimes, os, times] +import bitworld/[decision_trajectory, native_http, native_stop, runtime, runtime_input] + +installNativeStopHandlers() +var control: NativeRequestControl +var captures: seq[RuntimeInputCapture] +let deadline = getMonoTime() + initDuration(seconds = 2) +proc input(value, source: string): string = + readRuntimeInput(value, source, deadline, control, 4096, 4096, captures) + +try: + let config = readRuntimeConfig(input) + discard parseJson(config.config) + echo "runtime config accepted" +finally: + writePrivate(getEnv("INPUT_PRIVATE_CAPTURE"), $runtimeInputCapturesJson(captures)) diff --git a/tests/support/runtime_input_probe.nim b/tests/support/runtime_input_probe.nim new file mode 100644 index 0000000..519203a --- /dev/null +++ b/tests/support/runtime_input_probe.nim @@ -0,0 +1,27 @@ +import std/[base64, json, monotimes, options, os, strutils, times] +import bitworld/[native_http, native_stop, runtime_input] + +let args = commandLineParams() +installNativeStopHandlers() +var control: NativeRequestControl +if args[1] == "prestop": requestNativeStop() +if args[1] == "precancel": control.cancelNativeRequest() +if args[1] == "presignal": + echo "installed" + flushFile(stdout) + discard stdin.readLine() +let deadline = getMonoTime() + initDuration(milliseconds = args[2].parseInt()) +let response = performInputGet(args[0], @[], deadline, control, + args[3].parseInt(), args[4].parseInt()) +var nextKind = "not_requested" +if args[1] == "twice": + let next = performInputGet(args[0] & "/second", @[], deadline, control, + args[3].parseInt(), args[4].parseInt()) + doAssert next.latencyMs.isNone + nextKind = $next.kind +var status = newJNull() +if response.httpStatus.isSome: status = %response.httpStatus.get() +echo $(%*{"kind": $response.kind, "status": status, + "body_b64": encode(response.bodyBytes), "headers_b64": encode(response.headerBytes), + "complete": response.transferComplete, "joined": response.responseReaderJoined, + "next_kind": nextKind}) diff --git a/tests/test_runtime.nim b/tests/test_runtime.nim index 67500ac..7e946d6 100644 --- a/tests/test_runtime.nim +++ b/tests/test_runtime.nim @@ -1,6 +1,13 @@ import - std/[os, strutils], - bitworld/runtime + std/[os, strutils, monotimes, times], + bitworld/[runtime, runtime_input, native_http] + +var inputControl: NativeRequestControl +var captures: seq[RuntimeInputCapture] +let inputDeadline = getMonoTime() + initDuration(seconds = 10) +proc readInput(value, source: string): string = + readRuntimeInput(value, source, inputDeadline, inputControl, 16 * 1024 * 1024, + 64 * 1024, captures) let workspace = getTempDir() / ("bitworld-cogame-runtime-" & $getCurrentProcessId()) if dirExists(workspace): @@ -12,7 +19,8 @@ let configPath = workspace / "config.json" writeFile(configPath, """{"config":true}""") doAssert readCogameUri( "file://" & configPath, - CogameConfigUriEnv + CogameConfigUriEnv, + readInput ) == """{"config":true}""" writeCogameUri( @@ -53,7 +61,7 @@ putEnv(CogameLoadReplayUriEnv, "file://" & replayPath) putEnv(CogameLogUriEnv, "file://" & logPath) writeFile(replayPath, "replay-bytes") -let runtimeConfig = readRuntimeConfig() +let runtimeConfig = readRuntimeConfig(readInput) doAssert runtimeConfig.host == "127.0.0.1" doAssert runtimeConfig.port == 9001 doAssert runtimeConfig.config == """{"config":true}""" diff --git a/tests/test_runtime_config_input.py b/tests/test_runtime_config_input.py new file mode 100644 index 0000000..4555eec --- /dev/null +++ b/tests/test_runtime_config_input.py @@ -0,0 +1,45 @@ +"""Runtime config input records real bytes before status, UTF-8, or JSON rejection.""" +import base64 +import http.server +import json +import os +import subprocess +import sys +import tempfile +import threading +from pathlib import Path + +probe = Path(sys.argv[1]).resolve() +with tempfile.TemporaryDirectory(prefix="bitworld-input-private-") as directory: + root = Path(directory) + for mode in ("valid", "status", "utf8", "json"): + raw = b"\xffinvalid" if mode == "utf8" else b'{"invalid"' if mode == "json" else b'{"secret":"PRIVATE_INPUT_SENTINEL"}' + class Fixture(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(503 if mode == "status" else 200) + self.send_header("Content-Length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + def log_message(self, *_args): + pass + with http.server.ThreadingHTTPServer(("127.0.0.1", 0), Fixture) as server: + owner = threading.Thread(target=server.serve_forever) + owner.start() + destination = root / f"{mode}.json" + try: + result = subprocess.run([str(probe)], capture_output=True, text=True, + env=os.environ | {"COGAME_CONFIG_URI": f"http://127.0.0.1:{server.server_port}/config", + "INPUT_PRIVATE_CAPTURE": str(destination)}, timeout=4) + assert (result.returncode == 0) == (mode == "valid") + capture = json.loads(destination.read_text()) + assert len(capture) == 1 + transport = capture[0]["transport"] + assert base64.b64decode(transport["response_body_b64"], validate=True) == raw + assert transport["response_complete"] and transport["response_reader_joined"] + assert transport["http_status"] == (503 if mode == "status" else 200) + assert destination.stat().st_mode & 0o777 == 0o600 + print(mode, "exact private input capture precedes validation", flush=True) + finally: + server.shutdown() + owner.join(timeout=3) + assert not owner.is_alive() diff --git a/tests/test_runtime_input.py b/tests/test_runtime_input.py new file mode 100644 index 0000000..4c380f9 --- /dev/null +++ b/tests/test_runtime_input.py @@ -0,0 +1,94 @@ +"""Bounded real startup GET ownership, partial bytes, and one shared deadline.""" +import base64 +import http.server +import json +import signal +import subprocess +import sys +import threading +import time +from pathlib import Path + +probe = Path(sys.argv[1]).resolve() +for mode in ("complete", "status", "redirect", "bytes", "headers", "deadline", "twice", + "TERM", "INT", "header-TERM", "prestop", "precancel", "presignal"): + entered = threading.Event() + release = threading.Event() + observed = [] + + class Fixture(http.server.BaseHTTPRequestHandler): + def do_GET(self): + observed.append(self.path) + if mode == "header-TERM": + self.wfile.write(b"HTTP/1.1 200 OK\r\nX-Partial: retained\r\n") + self.wfile.flush() + entered.set() + release.wait(3) + return + body = b"\xff\x00exact" if mode == "complete" else b"0123456789" + self.send_response(503 if mode == "status" else 302 if mode == "redirect" else 200) + self.send_header("X-Fixture", "first") + self.send_header("X-Fixture", "second") + self.send_header("Content-Length", str(len(body))) + self.send_header("Location", "/must-not-follow") + self.end_headers() + self.wfile.write(body[:2] if mode in ("deadline", "twice", "TERM", "INT") else body) + self.wfile.flush() + entered.set() + if mode in ("deadline", "twice", "TERM", "INT"): + release.wait(3) + + def log_message(self, *_args): + pass + + with http.server.ThreadingHTTPServer(("127.0.0.1", 0), Fixture) as server: + owner = threading.Thread(target=server.serve_forever) + owner.start() + process = subprocess.Popen([str(probe), f"http://127.0.0.1:{server.server_port}/input", + mode, "250" if mode in ("deadline", "twice") else "2000", + "5" if mode == "bytes" else "1024", + "40" if mode == "headers" else "4096"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) + started = time.monotonic() + try: + if mode == "presignal": + assert process.stdout.readline().strip() == "installed" + process.send_signal(signal.SIGTERM) + stdout, stderr = process.communicate("continue\n", timeout=3) + else: + if mode in ("TERM", "INT", "header-TERM"): + assert entered.wait(2) + time.sleep(.03) + process.send_signal(signal.SIGINT if mode == "INT" else signal.SIGTERM) + stdout, stderr = process.communicate(timeout=3) + assert process.returncode == 0, stderr + result = json.loads(stdout) + body = base64.b64decode(result["body_b64"], validate=True) + headers = base64.b64decode(result["headers_b64"], validate=True) + if mode in ("prestop", "precancel", "presignal"): + assert not observed and not body and not headers and result["joined"] is None + assert result["kind"] == ("nhCanceled" if mode == "precancel" else "nhInterrupted") + else: + assert result["joined"] is True and observed == ["/input"] + if mode in ("deadline", "twice", "TERM", "INT", "header-TERM"): + assert not result["complete"] + assert result["kind"] == ("nhDeadline" if mode in ("deadline", "twice") else "nhInterrupted") + assert body == (b"" if mode == "header-TERM" else b"01") + if mode == "header-TERM": assert b"X-Partial: retained\r\n" in headers + if mode == "twice": assert result["next_kind"] == "nhDeadline" + elif mode in ("bytes", "headers"): + assert result["kind"] == "nhLimitExceeded" and not result["complete"] + assert len(body) <= 5 if mode == "bytes" else len(headers) <= 40 + else: + assert result["kind"] == "nhComplete" and result["complete"] + assert result["status"] == (503 if mode == "status" else 302 if mode == "redirect" else 200) + assert body == (b"\xff\x00exact" if mode == "complete" else b"0123456789") + print(mode, result["kind"], round(time.monotonic() - started, 3), flush=True) + finally: + if process.poll() is None: process.terminate() + process.wait(timeout=3) + release.set() + server.shutdown() + owner.join(timeout=3) + assert not owner.is_alive() diff --git a/tests/test_runtime_input_https.py b/tests/test_runtime_input_https.py new file mode 100644 index 0000000..d7a588f --- /dev/null +++ b/tests/test_runtime_input_https.py @@ -0,0 +1,110 @@ +"""Verify configured private CA trust without changing global certificate stores.""" +import base64 +import http.server +import json +import os +import signal +import socket +import ssl +import subprocess +import sys +import tempfile +import threading +from pathlib import Path + +probe = Path(sys.argv[1]).resolve() +with tempfile.TemporaryDirectory(prefix="bitworld-https-") as directory: + root = Path(directory) + subprocess.run(["openssl", "req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", + "-nodes", "-days", "1", "-subj", "/CN=Owned fixture CA", "-keyout", str(root / "ca.key"), + "-out", str(root / "ca.pem")], check=True, capture_output=True) + subprocess.run(["openssl", "req", "-new", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", + "-nodes", "-subj", "/CN=localhost", "-keyout", str(root / "server.key"), + "-out", str(root / "server.csr")], check=True, capture_output=True) + (root / "extensions").write_text("subjectAltName=DNS:localhost\n") + subprocess.run(["openssl", "x509", "-req", "-in", str(root / "server.csr"), "-CA", str(root / "ca.pem"), + "-CAkey", str(root / "ca.key"), "-CAcreateserial", "-days", "1", "-extfile", str(root / "extensions"), + "-out", str(root / "server.pem")], check=True, capture_output=True) + observed = [] + + class Fixture(http.server.BaseHTTPRequestHandler): + def upload(self): + observed.append((self.command, self.rfile.read(int(self.headers["Content-Length"])))) + self.send_response(200) + self.send_header("Content-Length", "2") + self.end_headers() + self.wfile.write(b"ok") + + def do_GET(self): + observed.append((self.command, b"")) + self.send_response(200) + self.send_header("Content-Length", "2") + self.end_headers() + self.wfile.write(b"ok") + + def log_message(self, *_args): + pass + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(root / "server.pem", root / "server.key") + with http.server.ThreadingHTTPServer(("127.0.0.1", 0), Fixture) as server: + server.socket = context.wrap_socket(server.socket, server_side=True) + owner = threading.Thread(target=server.serve_forever) + owner.start() + try: + for transport in ("input",): + for trust in ("untrusted", "trusted", "wrong-host"): + env = {**os.environ, "NO_PROXY": "localhost,127.0.0.1"} + env.pop("SSL_CERT_FILE", None) + if trust != "untrusted": env["SSL_CERT_FILE"] = str(root / "ca.pem") + host = "127.0.0.1" if trust == "wrong-host" else "localhost" + process = subprocess.run([str(probe), f"https://{host}:{server.server_port}/", transport, "2000", "1024", "4096"], + env=env, capture_output=True, text=True, timeout=4) + assert process.returncode == 0, process.stderr + result = json.loads(process.stdout) + if trust == "trusted": + assert result["kind"] == "nhComplete" and result["complete"] and result["status"] == 200 + assert result["joined"] is True and base64.b64decode(result["body_b64"]) == b"ok" + else: + assert result["kind"] == "nhTransportFailure" and not result["complete"] + assert result["joined"] is True and result["status"] is None and result["body_b64"] == "" + print(transport, trust, result["kind"], flush=True) + assert observed == [("GET", b"")] + finally: + server.shutdown() + owner.join(timeout=4) + assert not owner.is_alive() + +# A real unfinished TLS handshake must join on either an absolute deadline or SIGTERM. +for mode in ("deadline", "TERM"): + entered = threading.Event() + release = threading.Event() + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + def hold_handshake(): + with listener.accept()[0] as connection: + assert connection.recv(4096) + entered.set() + release.wait(3) + owner = threading.Thread(target=hold_handshake) + owner.start() + process = subprocess.Popen([str(probe), f"https://127.0.0.1:{listener.getsockname()[1]}/", + mode, "250" if mode == "deadline" else "2000", "1024", "4096"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + assert entered.wait(2) + if mode == "TERM": process.send_signal(signal.SIGTERM) + stdout, stderr = process.communicate(timeout=3) + assert process.returncode == 0, stderr + result = json.loads(stdout) + assert result["kind"] == ("nhDeadline" if mode == "deadline" else "nhInterrupted") + assert result["joined"] is True and not result["complete"] + assert result["status"] is None and result["body_b64"] == result["headers_b64"] == "" + print("partial-TLS", mode, result["kind"], flush=True) + finally: + if process.poll() is None: process.terminate() + process.wait(timeout=3) + release.set() + owner.join(timeout=3) + assert not owner.is_alive()