Own startup input deadlines and private transport capture

.github/workflows/decision-trajectories.yml

@@ -3,9 +3,9 @@ name: Private decision trajectories
 on:
   push:
     branches: [master]
-    paths: [src/bitworld/native_websocket.nim, tests/support/native_websocket_probe.nim, tests/test_native_websocket.py, src/bitworld/decision_trajectory.nim, src/bitworld/runtime.nim, src/bitworld/native_http.nim, src/bitworld/native_stop.nim, src/bitworld/artifact_runtime.nim, tests/test_artifact_http.py, tests/support/artifact_http_probe.nim, tests/test_native_https.py, tests/support/native_https_probe.nim, tests/test_decision_trajectory.nim, tests/test_native_http.py, tests/support/native_http_probe.nim, tests/support/native_request_control_probe.nim, tests/test_native_request_control.py, .github/workflows/decision-trajectories.yml]
+    paths: [src/bitworld/runtime_input.nim, tests/test_runtime.nim, tests/support/runtime_input_probe.nim, tests/support/runtime_config_input_probe.nim, tests/test_runtime_input.py, tests/test_runtime_input_https.py, tests/test_runtime_config_input.py, src/bitworld/native_websocket.nim, tests/support/native_websocket_probe.nim, tests/test_native_websocket.py, src/bitworld/decision_trajectory.nim, src/bitworld/runtime.nim, src/bitworld/native_http.nim, src/bitworld/native_stop.nim, src/bitworld/artifact_runtime.nim, tests/test_artifact_http.py, tests/support/artifact_http_probe.nim, tests/test_native_https.py, tests/support/native_https_probe.nim, tests/test_decision_trajectory.nim, tests/test_native_http.py, tests/support/native_http_probe.nim, tests/support/native_request_control_probe.nim, tests/test_native_request_control.py, .github/workflows/decision-trajectories.yml]
   pull_request:
-    paths: [src/bitworld/native_websocket.nim, tests/support/native_websocket_probe.nim, tests/test_native_websocket.py, src/bitworld/decision_trajectory.nim, src/bitworld/runtime.nim, src/bitworld/native_http.nim, src/bitworld/native_stop.nim, src/bitworld/artifact_runtime.nim, tests/test_artifact_http.py, tests/support/artifact_http_probe.nim, tests/test_native_https.py, tests/support/native_https_probe.nim, tests/test_decision_trajectory.nim, tests/test_native_http.py, tests/support/native_http_probe.nim, tests/support/native_request_control_probe.nim, tests/test_native_request_control.py, .github/workflows/decision-trajectories.yml]
+    paths: [src/bitworld/runtime_input.nim, tests/test_runtime.nim, tests/support/runtime_input_probe.nim, tests/support/runtime_config_input_probe.nim, tests/test_runtime_input.py, tests/test_runtime_input_https.py, tests/test_runtime_config_input.py, src/bitworld/native_websocket.nim, tests/support/native_websocket_probe.nim, tests/test_native_websocket.py, src/bitworld/decision_trajectory.nim, src/bitworld/runtime.nim, src/bitworld/native_http.nim, src/bitworld/native_stop.nim, src/bitworld/artifact_runtime.nim, tests/test_artifact_http.py, tests/support/artifact_http_probe.nim, tests/test_native_https.py, tests/support/native_https_probe.nim, tests/test_decision_trajectory.nim, tests/test_native_http.py, tests/support/native_http_probe.nim, tests/support/native_request_control_probe.nim, tests/test_native_request_control.py, .github/workflows/decision-trajectories.yml]
 
 jobs:
   test:
@@ -53,3 +53,11 @@ jobs:
         run: |
           nim c -d:release --threads:on --mm:orc --path:src -o:/tmp/native-websocket-probe tests/support/native_websocket_probe.nim
           python3 tests/test_native_websocket.py /tmp/native-websocket-probe
+      - name: Verify bounded startup inputs and private capture before validation
+        run: |
+          nim c -r --parallelBuild:1 --threads:on --mm:orc --path:src -o:/tmp/test-runtime tests/test_runtime.nim
+          nim c --parallelBuild:1 -d:release --threads:on --mm:orc --path:src -o:/tmp/runtime-input-probe tests/support/runtime_input_probe.nim
+          python3 tests/test_runtime_input.py /tmp/runtime-input-probe
+          python3 tests/test_runtime_input_https.py /tmp/runtime-input-probe
+          nim c --parallelBuild:1 -d:release --threads:on --mm:orc --path:src -o:/tmp/runtime-config-input-probe tests/support/runtime_config_input_probe.nim
+          python3 tests/test_runtime_config_input.py /tmp/runtime-config-input-probe

README.md

@@ -45,6 +45,26 @@ This makes the game world useful as a sandbox for questions like:
 - What incentives cause betrayal?
 - How do agents adapt to repeated social interaction?
 
+## Coworld startup inputs
+
+`readRuntimeConfig`, `readCogameUri`, and `readCogameEnv` require an
+`InputReader`. Native games supply a closure calling
+`runtime_input.readRuntimeInput` with one absolute startup deadline, an owned
+`NativeRequestControl`, body/header byte limits, and private captures. Reserve
+the game's cleanup budget before choosing that deadline.
+
+The reader owns each HTTP(S) handle until it joins. It retains received bytes
+before status, UTF-8, or game configuration validation. Redirects are rejected;
+TLS certificate and hostname verification remain enabled. `SSL_CERT_FILE`
+selects an explicit process trust bundle. File inputs must be regular files
+and obey the same byte limit and deadline.
+
+`runtimeInputCapturesJson` contains private source URIs and raw bytes. Keep it
+in the private checkpoint, never public replay or process logs. Games seal
+failed or interrupted initialization only after input ownership ends, using
+the original bounded cleanup deadline. `pathFromCogameUri` decodes local file
+URIs; it never downloads inputs.
+
 ## Visual Style
 
 Bit World is designed around strict retro display constraints:

src/bitworld/native_http.nim

@@ -13,9 +13,9 @@ type
   ArtifactHttpMethod* = enum
     ahPut = "PUT", ahPost = "POST"
   RequestPurpose = enum
-    rpInference, rpArtifact
+    rpInference, rpInput, rpArtifact
   NativeHttpKind* = enum
-    nhComplete, nhDeadline, nhInterrupted, nhCanceled, nhTransportFailure
+    nhComplete, nhDeadline, nhInterrupted, nhCanceled, nhTransportFailure, nhLimitExceeded
   NativeRequestControl* = object
     canceled: Atomic[bool]
   NativeHttpResponse* = object
@@ -31,6 +31,8 @@ type
     purpose: RequestPurpose
     control: ptr NativeRequestControl
     headerBytes, bodyBytes: string
+    maxHeaderBytes, maxBodyBytes: int
+    limitExceeded: bool
 
 # The pinned Nim binding omits these existing libcurl options.
 const
@@ -59,30 +61,43 @@ proc receiveHeaders(buffer: cstring, size, count: int, context: pointer): int {.
   let transfer = cast[ptr Transfer](context)
   result = size * count
   let offset = transfer.headerBytes.len
+  if result > transfer.maxHeaderBytes - offset:
+    let retained = transfer.maxHeaderBytes - offset
+    transfer.headerBytes.setLen(offset + retained)
+    if retained > 0: copyMem(transfer.headerBytes[offset].addr, buffer, retained)
+    transfer.limitExceeded = true
+    return 0
   transfer.headerBytes.setLen(offset + result)
   if result > 0: copyMem(transfer.headerBytes[offset].addr, buffer, result)
 
 proc receiveBody(buffer: cstring, size, count: int, context: pointer): int {.cdecl.} =
   let transfer = cast[ptr Transfer](context)
   result = size * count
   let offset = transfer.bodyBytes.len
+  if result > transfer.maxBodyBytes - offset:
+    let retained = transfer.maxBodyBytes - offset
+    transfer.bodyBytes.setLen(offset + retained)
+    if retained > 0: copyMem(transfer.bodyBytes[offset].addr, buffer, retained)
+    transfer.limitExceeded = true
+    return 0
   transfer.bodyBytes.setLen(offset + result)
   if result > 0: copyMem(transfer.bodyBytes[offset].addr, buffer, result)
 
 proc checkTransfer(context: pointer, downloadTotal, downloaded,
     uploadTotal, uploaded: int64): cint {.cdecl.} =
   let transfer = cast[ptr Transfer](context)
-  if (transfer.purpose == rpInference and
+  if (transfer.purpose != rpArtifact and
       (interruptionRequested() or transfer.control[].nativeRequestCanceled())) or
       getMonoTime() >= transfer.deadline: 1 else: 0
 
-proc performOwnedRequest(url: string, httpMethod: ArtifactHttpMethod,
+proc performOwnedRequest(url: string, httpMethod: string,
     headers: HttpHeaders, body: string, deadline: MonoTime,
-    purpose: RequestPurpose, control: var NativeRequestControl): NativeHttpResponse =
-  if purpose == rpInference and interruptionRequested():
+    purpose: RequestPurpose, control: var NativeRequestControl,
+    maxBodyBytes, maxHeaderBytes: int): NativeHttpResponse =
+  if purpose != rpArtifact and interruptionRequested():
     result.kind = nhInterrupted
     return
-  if purpose == rpInference and control.nativeRequestCanceled():
+  if purpose != rpArtifact and control.nativeRequestCanceled():
     result.kind = nhCanceled
     return
   let remaining = (deadline - getMonoTime()).inNanoseconds
@@ -93,7 +108,8 @@ proc performOwnedRequest(url: string, httpMethod: ArtifactHttpMethod,
   let handle = easy_init()
   doAssert handle != nil, "Cannot allocate native HTTP handle"
   var headerList: Pslist
-  var transfer = Transfer(deadline: deadline, purpose: purpose, control: control.addr)
+  var transfer = Transfer(deadline: deadline, purpose: purpose, control: control.addr,
+    maxBodyBytes: maxBodyBytes, maxHeaderBytes: maxHeaderBytes)
   var oldMask, pipeMask, previousPending: Sigset
   doAssert sigemptyset(pipeMask) == 0
   doAssert sigaddset(pipeMask, SIGPIPE) == 0
@@ -107,9 +123,10 @@ proc performOwnedRequest(url: string, httpMethod: ArtifactHttpMethod,
       doAssert appended != nil, "Cannot allocate native HTTP headers"
       headerList = appended
     requireCurl(handle.easy_setopt(OPT_URL, url.cstring))
-    requireCurl(handle.easy_setopt(OPT_CUSTOMREQUEST, ($httpMethod).cstring))
-    requireCurl(handle.easy_setopt(OPT_POSTFIELDS, body.cstring))
-    requireCurl(handle.easy_setopt(OPT_POSTFIELDSIZE, clong(body.len)))
+    requireCurl(handle.easy_setopt(OPT_CUSTOMREQUEST, httpMethod.cstring))
+    if httpMethod != "GET":
+      requireCurl(handle.easy_setopt(OPT_POSTFIELDS, body.cstring))
+      requireCurl(handle.easy_setopt(OPT_POSTFIELDSIZE, clong(body.len)))
     requireCurl(handle.easy_setopt(OPT_HTTPHEADER, headerList))
     requireCurl(handle.easy_setopt(OPT_FOLLOWLOCATION, clong(0)))
     requireCurl(handle.easy_setopt(OptProtocols, clong(3))) # HTTP and HTTPS only.
@@ -125,10 +142,10 @@ proc performOwnedRequest(url: string, httpMethod: ArtifactHttpMethod,
     requireCurl(handle.easy_setopt(OptXferInfoFunction, checkTransfer))
     let started = getMonoTime()
     let finalRemaining = (deadline - started).inNanoseconds
-    if finalRemaining <= 0 or (purpose == rpInference and
+    if finalRemaining <= 0 or (purpose != rpArtifact and
         (interruptionRequested() or control.nativeRequestCanceled())):
-      result.kind = if purpose == rpInference and interruptionRequested(): nhInterrupted
-        elif purpose == rpInference and control.nativeRequestCanceled(): nhCanceled
+      result.kind = if purpose != rpArtifact and interruptionRequested(): nhInterrupted
+        elif purpose != rpArtifact and control.nativeRequestCanceled(): nhCanceled
         else: nhDeadline
       return
     let milliseconds = clong((finalRemaining + 999_999) div 1_000_000)
@@ -140,8 +157,9 @@ proc performOwnedRequest(url: string, httpMethod: ArtifactHttpMethod,
     var status: clong
     requireCurl(handle.easy_getinfo(INFO_RESPONSE_CODE, status.addr))
     if status != 0: result.httpStatus = some(int(status))
-    if purpose == rpInference and interruptionRequested(): result.kind = nhInterrupted
-    elif purpose == rpInference and control.nativeRequestCanceled(): result.kind = nhCanceled
+    if purpose != rpArtifact and interruptionRequested(): result.kind = nhInterrupted
+    elif purpose != rpArtifact and control.nativeRequestCanceled(): result.kind = nhCanceled
+    elif transfer.limitExceeded: result.kind = nhLimitExceeded
     elif code == E_OPERATION_TIMEOUTED or getMonoTime() >= deadline: result.kind = nhDeadline
     elif code == E_OK: result.kind = nhComplete
     else: result.kind = nhTransportFailure
@@ -157,8 +175,8 @@ proc performOwnedRequest(url: string, httpMethod: ArtifactHttpMethod,
         doAssert sigwait(pipeMask, received) == 0
     var discardedMask: Sigset
     doAssert pthread_sigmask(SIG_SETMASK, oldMask, discardedMask) == 0
-  if purpose == rpInference and interruptionRequested(): result.kind = nhInterrupted
-  elif purpose == rpInference and control.nativeRequestCanceled(): result.kind = nhCanceled
+  if purpose != rpArtifact and interruptionRequested(): result.kind = nhInterrupted
+  elif purpose != rpArtifact and control.nativeRequestCanceled(): result.kind = nhCanceled
   elif getMonoTime() >= deadline: result.kind = nhDeadline
   result.headerBytes = move transfer.headerBytes
   result.bodyBytes = move transfer.bodyBytes
@@ -167,11 +185,20 @@ proc performOwnedRequest(url: string, httpMethod: ArtifactHttpMethod,
 proc performNativePost*(url: string, headers: HttpHeaders, body: string,
     deadline: MonoTime, control: var NativeRequestControl): NativeHttpResponse =
   ## A caller shares one deadline across retries. Never reset it per attempt.
-  performOwnedRequest(url, ahPost, headers, body, deadline, rpInference, control)
+  performOwnedRequest(url, "POST", headers, body, deadline, rpInference, control,
+    int.high, int.high)
 
 proc performArtifactUpload*(url: string, httpMethod: ArtifactHttpMethod,
     headers: HttpHeaders, body: string, cleanupDeadline: MonoTime): NativeHttpResponse =
   ## Checkpoint finalization has its own finite cleanup lifetime after inference stops.
   ## No caller can disable interruption in the inference API.
   var control: NativeRequestControl
-  performOwnedRequest(url, httpMethod, headers, body, cleanupDeadline, rpArtifact, control)
+  performOwnedRequest(url, $httpMethod, headers, body, cleanupDeadline, rpArtifact, control,
+    int.high, int.high)
+
+proc performInputGet*(url: string, headers: HttpHeaders, deadline: MonoTime,
+    control: var NativeRequestControl, maxBodyBytes, maxHeaderBytes: int): NativeHttpResponse =
+  ## Startup input has one owner/deadline and never follows a redirect to a new URI.
+  doAssert maxBodyBytes > 0 and maxHeaderBytes > 0
+  performOwnedRequest(url, "GET", headers, "", deadline, rpInput, control,
+    maxBodyBytes, maxHeaderBytes)

src/bitworld/runtime.nim

@@ -1,5 +1,5 @@
 import
-  std/[os, parseopt, strutils],
+  std/[os, parseopt, strutils, unicode],
   curly
 
 const
@@ -14,6 +14,8 @@ const
   CogamePortEnv* = "COGAME_PORT"
 
 type
+  InputReader* = proc(value, source: string): string {.closure.}
+
   CogameRuntimeError* = object of CatchableError
 
   RuntimeConfig* = object
@@ -94,52 +96,23 @@ proc isHttpCogameUri*(value: string): bool =
   ## Returns true when a Coworld URI is an HTTP(S) URI.
   value.startsWith("http://") or value.startsWith("https://")
 
-proc readCogameUri*(value, source: string): string =
-  ## Reads data from a Coworld file URI or HTTP(S) signed URI.
-  if value.len == 0:
-    return ""
+proc readCogameUri*(value, source: string, inputReader: InputReader): string =
+  ## The caller owns input deadlines, cancellation, limits, and private capture.
+  if value.len == 0: return ""
+  inputReader(value, source)
 
-  let path = filePathFromCogameUri(value, source)
-  if path.len > 0:
-    return readFile(path)
-
-  if value.isHttpCogameUri():
-    let client = newCurlPool(1)
-    defer: client.close()
-    let response = client.get(value)
-    if response.code < 200 or response.code >= 300:
-      raise newException(
-        IOError,
-        source & " download failed: " & $response.code
-      )
-    return response.body
-
-  if "://" in value:
-    raise newException(
-      CogameRuntimeError,
-      "unsupported URI from " & source & ": " & value
-    )
-
-  raise newException(CogameRuntimeError, source & " must be a URI")
-
-proc readCogameEnv*(name: string): string =
-  ## Reads data from a Coworld URI environment variable.
-  readCogameUri(getEnv(name), name)
+proc readCogameEnv*(name: string, inputReader: InputReader): string =
+  readCogameUri(getEnv(name), name, inputReader)
 
 proc pathFromCogameUri*(value, source: string): string =
-  ## Converts a Coworld file/input URI into a local path.
+  ## Converts a Coworld local file URI into a path without performing transport.
   if value.len == 0:
     return ""
 
   let path = filePathFromCogameUri(value, source)
   if path.len > 0:
     return path
 
-  if value.isHttpCogameUri():
-    result = getTempDir() / ("cogame-" & source.toLowerAscii())
-    writeFile(result, readCogameUri(value, source))
-    return
-
   if "://" in value:
     raise newException(
       CogameRuntimeError,
@@ -276,7 +249,7 @@ proc writeRuntimeTarget(
     return
   value.writeLocalTarget(data)
 
-proc readRuntimeConfig*(): RuntimeConfig =
+proc readRuntimeConfig*(inputReader: InputReader): RuntimeConfig =
   ## Reads the Coworld runtime config from CLI arguments and env vars.
   result = RuntimeConfig(host: RuntimeDefaultHost, port: RuntimeDefaultPort)
   var
@@ -308,11 +281,11 @@ proc readRuntimeConfig*(): RuntimeConfig =
         configSet = true
       of "config-path":
         key.requireValue(val)
-        result.config = readFile(val)
+        result.config = inputReader("file://" & absolutePath(val), "--" & key)
         configSet = true
       of "config-uri":
         key.requireValue(val)
-        result.config = readCogameUri(val, "--" & key)
+        result.config = readCogameUri(val, "--" & key, inputReader)
         configSet = true
       of "results":
         key.requireValue(val)
@@ -332,12 +305,12 @@ proc readRuntimeConfig*(): RuntimeConfig =
         saveReplaySet = true
       of "load-replay":
         key.requireValue(val)
-        result.replay = readFile(val)
+        result.replay = inputReader("file://" & absolutePath(val), "--" & key)
         result.replayMode = true
         loadReplaySet = true
       of "load-replay-uri":
         key.requireValue(val)
-        result.replay = readCogameUri(val, "--" & key)
+        result.replay = readCogameUri(val, "--" & key, inputReader)
         result.replayMode = true
         loadReplaySet = true
       of "log":
@@ -383,18 +356,20 @@ proc readRuntimeConfig*(): RuntimeConfig =
   if not configSet:
     let configUri = getEnv(CogameConfigUriEnv)
     if configUri.len > 0:
-      result.config = readCogameUri(configUri, CogameConfigUriEnv)
+      result.config = readCogameUri(configUri, CogameConfigUriEnv, inputReader)
   if not resultsSet:
     result.resultsUri = getEnv(CogameResultsUriEnv)
   if not saveReplaySet:
     result.replayUri = getEnv(CogameSaveReplayUriEnv)
   if not loadReplaySet:
     let replayUri = getEnv(CogameLoadReplayUriEnv)
     if replayUri.len > 0:
-      result.replay = readCogameUri(replayUri, CogameLoadReplayUriEnv)
+      result.replay = readCogameUri(replayUri, CogameLoadReplayUriEnv, inputReader)
       result.replayMode = true
   if not logSet:
     result.logUri = getEnv(CogameLogUriEnv)
+  if result.config.validateUtf8() != -1:
+    raise newException(CogameRuntimeError, "runtime config must be UTF-8")
 
 proc writeResults*(config: RuntimeConfig, data: string) =
   ## Writes a Coworld results artifact if a target is configured.