feat(remote): run a worker behind a remote wrapper that records its instance and outcome

app/Main.hs

@@ -12,6 +12,7 @@ import Invar.Worker qualified as W
 import Options qualified as O
 import PolicyInput qualified
 import Quality qualified
+import Remote qualified
 import Scoring qualified
 import System.Console.GetOpt (OptDescr, usageInfo)
 import System.Environment (getArgs)
@@ -35,6 +36,7 @@ main = do
         "use" : rest -> Use.run rest
         "train" : rest -> Training.run rest
         "policy" : rest -> PolicyInput.run rest
+        "remote" : rest -> Remote.run rest
         "infer" : "batch" : rest -> InferenceBatch.run rest
         "infer" : rest -> do
             (worker, planned, bound) <- parse rest
@@ -43,7 +45,7 @@ main = do
         _ -> die usage
 
 usage :: String
-usage = usageInfo "Usage: invar use prepare|plan|run|inspect|admit OPTIONS | invar infer OPTIONS | invar infer batch OPTIONS < calls.json | invar score OPTIONS | invar policy OPTIONS | invar train OPTIONS < tasks.json | invar evaluate OPTIONS < tasks.json | invar quality OPTIONS | invar inspect KIND OPTIONS | invar compare KIND OPTIONS\nUse the command's --help for its options. Inference selects --checkpoint, or an explicit --adapter with all four digest options. --checkpoint reads the published policy description. --worker-config is optional." options
+usage = usageInfo "Usage: invar use prepare|plan|run|inspect|admit OPTIONS | invar infer OPTIONS | invar infer batch OPTIONS < calls.json | invar score OPTIONS | invar policy OPTIONS | invar train OPTIONS < tasks.json | invar evaluate OPTIONS < tasks.json | invar quality OPTIONS | invar inspect KIND OPTIONS | invar compare KIND OPTIONS | invar remote run|status|terminate OPTIONS\nUse the command's --help for its options. Inference selects --checkpoint, or an explicit --adapter with all four digest options. --checkpoint reads the published policy description. --worker-config is optional." options
 
 options :: [OptDescr (String, String)]
 options = O.descriptions [("python", "Python executable"), ("worker", "Inference worker script"), ("worker-config", "Optional worker launch configuration"), ("cache", "Pinned model cache"), ("adapter", "Explicit adapter file or native handoff directory"), ("checkpoint", "Checkpoint containing policy.json and adapter.safetensors")] ++ InferenceInput.options

app/Remote.hs

@@ -0,0 +1,75 @@
+{-# LANGUAGE OverloadedStrings #-}
+
+module Remote (run) where
+
+import Control.Monad (unless)
+import Data.Aeson (encode, object, (.=))
+import Data.ByteString.Lazy.Char8 qualified as Lazy
+import Data.List (nub)
+import Data.Text qualified as Text
+import Invar.Remote qualified as R
+import Invar.Remote.Control qualified as C
+import Options qualified as O
+import System.Console.GetOpt (OptDescr, usageInfo)
+import System.Environment (getExecutablePath)
+import System.Exit (die, exitWith)
+
+run :: [String] -> IO ()
+run ["--help"] = putStrLn usage
+run [_, "--help"] = putStrLn usage
+run ("run" : supplied) = do
+    let (named, command) = break (== "--") supplied
+        (declared, rest) = environment named
+    (target, values) <- case drop 1 command of
+        chosen : remaining -> pure (chosen, remaining)
+        [] -> die "remote run needs -- PROGRAM [ARGUMENT]..."
+    fields <- either die pure (O.parse runOptions rest)
+    pairs <- either die pure (traverse assignment declared)
+    unless (length (nub (map fst pairs)) == length pairs) (die "Duplicate environment names are not allowed")
+    root <- either die pure (O.required fields "instance")
+    chosen <- either die pure (O.required fields "nonce")
+    self <- getExecutablePath
+    R.run self (R.Launch root (Text.pack chosen) (O.optional fields "directory") pairs target values) >>= exitWith
+run ("exec" : supplied) = do
+    let (named, command) = break (== "--") supplied
+    fields <- either die pure (O.parse execOptions named)
+    barrier <- either die pure (O.numeric fields "barrier")
+    report <- either die pure (O.numeric fields "status")
+    case drop 1 command of
+        target : values -> R.exec (fromInteger barrier) (fromInteger report) target values
+        [] -> die "remote exec needs -- PROGRAM [ARGUMENT]..."
+run ("status" : supplied) = do
+    fields <- either die pure (O.parse statusOptions supplied)
+    root <- either die pure (O.required fields "instance")
+    R.status root >>= either die (Lazy.putStrLn . encode . C.describe)
+run ("terminate" : supplied) = do
+    fields <- either die pure (O.parse terminateOptions supplied)
+    root <- either die pure (O.required fields "instance")
+    chosen <- either die pure (O.required fields "nonce")
+    request <- either die pure (C.Request (Text.pack chosen) <$> O.numeric fields "wrapper" <*> O.numeric fields "worker" <*> O.numeric fields "grace")
+    seconds <- either die pure (O.numeric fields "wait")
+    (written, answered, current) <- R.terminate root request seconds
+    Lazy.putStrLn (encode (object ["requested" .= written, "answer" .= C.answerName answered, "status" .= C.describe current]))
+run _ = die usage
+
+environment :: [String] -> ([String], [String])
+environment ("--environment" : value : rest) = let (declared, remaining) = environment rest in (value : declared, remaining)
+environment (other : rest) = fmap (other :) (environment rest)
+environment [] = ([], [])
+
+assignment :: String -> Either String (String, String)
+assignment value = case break (== '=') value of
+    (name@(_ : _), '=' : setting) -> Right (name, setting)
+    _ -> Left ("An environment entry is NAME=VALUE: " ++ value)
+
+runOptions, execOptions, statusOptions, terminateOptions :: [OptDescr (String, String)]
+runOptions = O.descriptions [("instance", "Instance directory to create; it must not exist"), ("nonce", "Nonce the core chose for this launch"), ("directory", "Working directory of the worker")]
+execOptions = O.descriptions [("barrier", "Descriptor to read the release from"), ("status", "Descriptor to report a failed start on")]
+statusOptions = O.descriptions [("instance", "Instance directory")]
+terminateOptions = O.descriptions [("instance", "Instance directory"), ("nonce", "Nonce of the instance"), ("wrapper", "Wrapper process id of the instance"), ("worker", "Worker process id of the instance"), ("grace", "Seconds between termination and kill"), ("wait", "Seconds to wait for the wrapper's confirmation")]
+
+usage :: String
+usage =
+    usageInfo
+        "Usage: invar remote run --instance DIRECTORY --nonce NONCE [--directory PATH] [--environment NAME=VALUE]... -- PROGRAM [ARGUMENT]...\n       invar remote status --instance DIRECTORY\n       invar remote terminate --instance DIRECTORY --nonce NONCE --wrapper PID --worker PID --grace SECONDS --wait SECONDS\nrun starts the worker behind a start barrier with exactly the declared environment, forwards its stdout unchanged, and writes its control files in the instance directory; status and terminate print one JSON object. invar remote exec is the internal child of run."
+        (runOptions ++ terminateOptions)

invar.cabal

@@ -28,29 +28,32 @@ library
   cc-options: -Wall -Wextra -Werror
   default-extensions: NoGeneralizedNewtypeDeriving
   hs-source-dirs: src
-  exposed-modules: Invar.Async.Plan, Invar.Tokenizer, Invar.Learn.Algorithm, Invar.Teacher, Invar.Async.Core, Invar.Async.Completion, Invar.Async.Entry, Invar.Async.Replay, Invar.Transcript, Invar.Canonical, Invar.Infer.Trajectory, Invar.Infer.Session, Invar.Infer.Replay, Invar.Resident.Owner, Invar.Journal, Invar.Runtime, Invar.Learn.Stream, Invar.Use.Measurement, Invar.Use.Decimal, Invar.Use.Confidence, Invar.Use.Plan, Invar.Use.Prepare, Invar.Use.Execution, Invar.Use.Statistics, Invar.Use, Invar.Score.Worker, Invar.Score.Resident, Invar.Score, Invar.Numerical, Invar.Spec.Request, Invar.Spec.Value, Invar.Spec.Program, Invar.Spec.Dependency, Invar.Spec.Operator, Invar.Spec.Evaluate, Invar.Spec.Artifact, Invar.Spec.Invocation, Invar.Spec.Evidence, Invar.Spec.Load, Invar.Construct, Invar.Literal, Invar.Store, Invar.Infer, Invar.Infer.Result, Invar.Infer.Observation, Invar.Infer.Invocation, Invar.Worker, Invar.Worker.Resident, Invar.Reward, Invar.Cohort, Invar.Schedule, Invar.Rollout, Invar.Learn, Invar.Learn.Advantage, Invar.Learn.Objective, Invar.Learn.Mismatch, Invar.Learn.Program, Invar.Learn.Wire, Invar.Learn.Protocol, Invar.Learn.Worker, Invar.Learn.Worker.Resident, Invar.Loop, Invar.Policy, Invar.Artifact, Invar.Workload, Invar.Evaluation, Invar.Quality, Invar.Learn.Report, Invar.Learn.Observation, Invar.Learn.State, Invar.History.Cohort, Invar.History.Trace, Invar.History.Generation, Invar.History.Schedule, Invar.History.Runtime, Invar.History.Initial, Invar.History
+  exposed-modules: Invar.Async.Plan, Invar.Tokenizer, Invar.Learn.Algorithm, Invar.Teacher, Invar.Async.Core, Invar.Async.Completion, Invar.Async.Entry, Invar.Async.Replay, Invar.Transcript, Invar.Canonical, Invar.Infer.Trajectory, Invar.Infer.Session, Invar.Infer.Replay, Invar.Resident.Owner, Invar.Journal, Invar.Runtime, Invar.Learn.Stream, Invar.Use.Measurement, Invar.Use.Decimal, Invar.Use.Confidence, Invar.Use.Plan, Invar.Use.Prepare, Invar.Use.Execution, Invar.Use.Statistics, Invar.Use, Invar.Score.Worker, Invar.Score.Resident, Invar.Score, Invar.Numerical, Invar.Spec.Request, Invar.Spec.Value, Invar.Spec.Program, Invar.Spec.Dependency, Invar.Spec.Operator, Invar.Spec.Evaluate, Invar.Spec.Artifact, Invar.Spec.Invocation, Invar.Spec.Evidence, Invar.Spec.Load, Invar.Construct, Invar.Literal, Invar.Store, Invar.Infer, Invar.Infer.Result, Invar.Infer.Observation, Invar.Infer.Invocation, Invar.Worker, Invar.Worker.Resident, Invar.Reward, Invar.Cohort, Invar.Schedule, Invar.Rollout, Invar.Learn, Invar.Learn.Advantage, Invar.Learn.Objective, Invar.Learn.Mismatch, Invar.Learn.Program, Invar.Learn.Wire, Invar.Learn.Protocol, Invar.Learn.Worker, Invar.Learn.Worker.Resident, Invar.Loop, Invar.Policy, Invar.Artifact, Invar.Workload, Invar.Evaluation, Invar.Quality, Invar.Learn.Report, Invar.Learn.Observation, Invar.Learn.State, Invar.History.Cohort, Invar.History.Trace, Invar.History.Generation, Invar.History.Schedule, Invar.History.Runtime, Invar.History.Initial, Invar.History, Invar.Remote, Invar.Remote.Control
   other-modules: Invar.Async.Completion.Internal, Invar.Infer.Trajectory.Internal, Invar.Infer.Records, Invar.Learn.Step, Invar.Spec.Measurement, Invar.Spec.Domain, Invar.Use.Encoding, Invar.Use.Contract, Invar.Use.Admission, Invar.Use.Finding, Invar.Numerical.Logarithm, Invar.Spec.UseContract, Invar.Spec.Obligation, Invar.Evidence.Encoding, Invar.Use.Observation, Invar.Spec.Use, Invar.Numerical.Distribution, Invar.Score.Execution, Invar.Score.Probe, Invar.Score.Resources, Invar.Spec.Score, Invar.Infer.Schema, Invar.Score.Program, Invar.Score.Output, Invar.Spec.Numerical, Invar.Quality.Summary, Invar.Reward.Decimal, Invar.Reward.Program, Invar.Materialization, Invar.Load, Invar.Infer.Load, Invar.Infer.Batch, Invar.Infer.Framing, Invar.Spec.Syntax, Invar.Spec.Decode, Invar.Spec.Encode, Invar.Construct.Types, Invar.Infer.Wire, Invar.Infer.Output, Invar.Infer.Model, Invar.Rollout.Internal, Invar.Rollout.Resident, Invar.Process, Invar.Process.Internal, Invar.Process.Resident, Invar.Resident, Invar.Resident.Group, Invar.Policy.Header, Invar.Policy.Description, Invar.Policy.Encoding, Invar.Json, Invar.Learn.Native, Invar.Learn.Observed, Invar.Learn.Codec, Invar.Learn.Adapter, Invar.Learn.Checkpoint, Invar.Learn.Checkpoint.Types, Invar.Learn.Checkpoint.MLX, Invar.Learn.Request, Invar.Learn.Gradient, Invar.Policy.File, Invar.Learn.Probability, Invar.Learn.Fingerprint, Invar.Float32, Invar.Digest, Invar.Learn.Trace, Invar.Learn.Report.Internal, Invar.Async.Recorded, Invar.Learn.Framing, Invar.Learn.Worker.Internal, Invar.Learn.Worker.Observation, Invar.Learn.Worker.Owner, Invar.Loop.Owner, Invar.History.Execution, Invar.History.Profile, Invar.History.Publication, Invar.History.Artifacts, Invar.Measurement.Duration
 
 test-suite request
   import: defaults
   type: exitcode-stdio-1.0
   hs-source-dirs: test
   main-is: Main.hs
-  other-modules: TokenizerFixture, Tokenizers, Algorithms, Distillations, Teachers, Events, Journals, Replays, Recoveries, Restarts, Comparisons, Drivers, Generation, Logs, Evaluations, Traces, Steps, UseBernstein, UsePlan, UsePrepare, UseExecution, UseGeneric, Statistics, UseAdmission, UseEvidence, Uses, UseFixture, NumericalProbes, ScoreProbes, NumericalScores, Scores, Numerical, Model, History, Properties, Fixtures, Arguments, Values, Dependencies, Evaluation, Artifacts, Construction, Collections, Records, Literals, Invocations, Evidence, Loads, Store, Checkpoints, Inference, Results, Calls, BatchCalls, GroupedCalls, BatchedProtocol, ResidentFixture, ResidentCalls, ResidentWorkloads, ResidentDrivers, ResidentObservations, Streams, Rewards, Cohorts, Rollouts, Sessions, Learning, Advantages, Objectives, Mismatches, Updates, LearnerFixture, LearnerCalls, Loops, Schedules, Policies, Probabilities, Workloads, Reports, Quality, Observations, InferenceObservations
+  other-modules: Remotes, TokenizerFixture, Tokenizers, Algorithms, Distillations, Teachers, Events, Journals, Replays, Recoveries, Restarts, Comparisons, Drivers, Generation, Logs, Evaluations, Traces, Steps, UseBernstein, UsePlan, UsePrepare, UseExecution, UseGeneric, Statistics, UseAdmission, UseEvidence, Uses, UseFixture, NumericalProbes, ScoreProbes, NumericalScores, Scores, Numerical, Model, History, Properties, Fixtures, Arguments, Values, Dependencies, Evaluation, Artifacts, Construction, Collections, Records, Literals, Invocations, Evidence, Loads, Store, Checkpoints, Inference, Results, Calls, BatchCalls, GroupedCalls, BatchedProtocol, ResidentFixture, ResidentCalls, ResidentWorkloads, ResidentDrivers, ResidentObservations, Streams, Rewards, Cohorts, Rollouts, Sessions, Learning, Advantages, Objectives, Mismatches, Updates, LearnerFixture, LearnerCalls, Loops, Schedules, Policies, Probabilities, Workloads, Reports, Quality, Observations, InferenceObservations
   build-depends:
     invar,
     hedgehog >= 1.7 && < 1.8,
     aeson >= 2.3.1 && < 2.4,
     text >= 2.1 && < 2.2,
     directory >= 1.3 && < 1.4,
     filepath >= 1.5 && < 1.6,
-    unix >= 2.8 && < 2.9
+    unix >= 2.8 && < 2.9,
+    process >= 1.6.26 && < 1.7,
+    cryptohash-sha256 >= 0.11.102.1 && < 0.12
+  build-tool-depends: invar:invar
 
 executable invar
   import: defaults
   hs-source-dirs: app
   main-is: Main.hs
-  other-modules: Use, UseRuns, UsePlan, UsePrepare, UseExecution, UseInput, UseRun, Check, Located, Envelope, Scoring, Numerical, Options, InferenceInput, EvaluationInput, InferenceBatch, PolicyInput, NativeCodec, HistoryInput, Dataset, Training, Evaluation, Quality, Inspection, Comparison
+  other-modules: Use, UseRuns, UsePlan, UsePrepare, UseExecution, UseInput, UseRun, Check, Located, Envelope, Scoring, Numerical, Options, InferenceInput, EvaluationInput, InferenceBatch, PolicyInput, NativeCodec, HistoryInput, Dataset, Training, Evaluation, Quality, Inspection, Comparison, Remote
   build-depends: invar, text >= 2.1 && < 2.2, aeson >= 2.3.1 && < 2.4, cryptohash-sha256 >= 0.11.102.1 && < 0.12, directory >= 1.3 && < 1.4, filepath >= 1.5 && < 1.6, process >= 1.6.26 && < 1.7
 
 source-repository head