SSH agent forwarding into local and remote devcontainers

package.json

@@ -188,9 +188,10 @@
   "scripts": {
     "compile": "tsc --noEmit && eslint src",
     "build": "npm run build:dev",
-    "build:dev": "node esbuild.js && cp src/remote/installServer.sh dist/ && cp src/engine/Dockerfile dist/",
+    "copy_templates": "cp src/remote/installServer.sh dist/ && cp src/engine/Dockerfile dist/ && cp src/remote/exec-pipe/remote.ts dist/",
+    "build:dev": "node esbuild.js && npm run copy_templates",
+    "build:prod": "node esbuild.js --production && npm run copy_templates",
     "build:clean": "rm -rf dist out",
-    "build:prod": "node esbuild.js --production && cp src/remote/installServer.sh dist/ && cp src/engine/Dockerfile dist/",
     "test": "npm run test:noengine",
     "test:noengine": "SKIP_ENGINE_TESTS=1 vitest run",
     "test:engine": "vitest run",

src/common/ctx/execCtx.ts

@@ -16,10 +16,20 @@ export interface FsCtx {
     write(filePath: vscode.Uri, content: string): Promise<void>,
 };
 
+export interface SpawnedProcess {
+    readonly stdin: { write(data: Buffer | Uint8Array): void, end(): void },
+    onStdout(cb: (data: Buffer) => void): void,
+    onStderr(cb: (data: Buffer) => void): void,
+    onError(cb: (err: Error) => void): void,
+    onExit(cb: (code: number | null) => void): void,
+    onClose(cb: () => void): void,
+}
+
 export interface ExecCtx {
     kind: ExecCtxKind,
     fs: FsCtx,
     run(cmdArgs: string[], opts: RunOpts): Promise<CmdResult>,
+    spawn(cmdArgs: string[], opts: RunOpts): Promise<SpawnedProcess>,
     getHostUserInfo(): Promise<HostUserInfo>,
     env(): Promise<vscode.ExecEnvironment>,
     getSettings(): Promise<Settings>,

src/common/ctx/localCtx.ts

@@ -1,8 +1,9 @@
 import * as vscode from "vscode";
+import * as chproc from "node:child_process";
 
-import { ExecCtx, ExecCtxKind, FsCtx } from "./execCtx";
+import { ExecCtx, ExecCtxKind, FsCtx, SpawnedProcess } from "./execCtx";
 import { HostUserInfo } from "../utils";
-import { formatCmdErr, spawn } from "../spawn";
+import { collectOutput, formatCmdErr } from "../spawn";
 import { getLogSink } from "../../extension/log";
 import { SpawnError } from "../../extension/error";
 import { CmdResult, RunOpts } from "../opts";
@@ -39,14 +40,37 @@ export class LocalExecCtx implements ExecCtx {
     }
 
     async run(cmdArgs: string[], opts: RunOpts): Promise<CmdResult> {
+        const proc = await this.spawn(cmdArgs, opts);
+        return collectOutput(proc, opts);
+    }
+
+    async spawn(cmdArgs: string[], opts: RunOpts): Promise<SpawnedProcess> {
         if (this.execServer) {
-            return await spawnRemote(this.execServer, cmdArgs, opts);
+            return spawnRemote(this.execServer, cmdArgs, opts);
         }
-        return await spawn(cmdArgs[0], cmdArgs.slice(1), { ...opts, log: getLogSink() });
+
+        const [cmd, ...args] = cmdArgs;
+        const proc = chproc.spawn(cmd, args, {
+            cwd: opts.cwd,
+            env: { ...process.env, BUILDKIT_PROGRESS: "plain", ...opts.env },
+            stdio: ["pipe", "pipe", "pipe"],
+        });
+
+        return {
+            stdin: {
+                write(data: Buffer | Uint8Array) { proc.stdin.write(data); },
+                end() { proc.stdin.end(); },
+            },
+            onStdout(cb) { proc.stdout.on("data", cb); },
+            onStderr(cb) { proc.stderr.on("data", cb); },
+            onError(cb) { proc.on("error", cb); },
+            onExit(cb) { proc.on("exit", cb); },
+            onClose(cb) { proc.on("close", cb); },
+        };
     }
 
     async getHostUserInfo(): Promise<HostUserInfo> {
-        const userName = await spawn("id", ["-nu"], { log: getLogSink() });
+        const userName = await this.run(["id", "-nu"], {});
         /* eslint-disable @typescript-eslint/no-non-null-assertion */
         if (userName.exit === 0) {
             return {

src/common/ctx/remoteCtx.ts

@@ -1,8 +1,8 @@
 import * as vscode from "vscode";
 
-import { ExecCtx, ExecCtxKind, FsCtx } from "./execCtx";
+import { ExecCtx, ExecCtxKind, FsCtx, SpawnedProcess } from "./execCtx";
 import { HostUserInfo } from "../utils";
-import { formatCmdErr } from "../spawn";
+import { collectOutput, formatCmdErr } from "../spawn";
 import { getLogSink } from "../../extension/log";
 import { SpawnError } from "../../extension/error";
 import { CmdResult, RunOpts } from "../opts";
@@ -54,6 +54,11 @@ export class RemoteExecCtx implements ExecCtx {
     }
 
     public async run(cmdArgs: string[], opts: RunOpts): Promise<CmdResult> {
+        const proc = await this.spawn(cmdArgs, opts);
+        return collectOutput(proc, opts);
+    }
+
+    public async spawn(cmdArgs: string[], opts: RunOpts): Promise<SpawnedProcess> {
         return spawnRemote(this.execServer, cmdArgs, opts);
     }
 
@@ -95,7 +100,7 @@ export class RemoteExecCtx implements ExecCtx {
     }
 }
 
-export async function spawnRemote(execServer: vscode.ExecServer, cmdArgs: string[], opts: RunOpts) {
+export async function spawnRemote(execServer: vscode.ExecServer, cmdArgs: string[], opts: RunOpts): Promise<SpawnedProcess> {
     cmdCount += 1;
     const cmdId = cmdCount;
 
@@ -117,41 +122,28 @@ export async function spawnRemote(execServer: vscode.ExecServer, cmdArgs: string
         env: { ...env, BUILDKIT_PROGRESS: "plain" },
     });
 
-    if (opts.stdin !== undefined) {
-        spawned.stdin.write(new TextEncoder().encode(opts.stdin));
-        spawned.stdin.end();
-    }
-
-    let stdout = "";
-    let stderr = "";
-    const stdoutDecoder = new TextDecoder("utf-8");
-    const stderrDecoder = new TextDecoder("utf-8");
-
-    spawned.stdout.onDidReceiveMessage((data: Uint8Array) => {
-        const chunk = stdoutDecoder.decode(data, { stream: true });
-        stdout += chunk;
-        log.debug(`RemoteExecCtx.stdout.onDidReceiveMessage[${cmdStr()}]: len ${chunk.length}, chunk: '${chunk.trim()}'`);
-    });
-
-    spawned.stderr.onDidReceiveMessage((data: Uint8Array) => {
-        const chunk = stderrDecoder.decode(data, { stream: true });
-        stderr += chunk;
-        log.debug(`RemoteExecCtx.stderr.onDidReceiveMessage[${cmdStr()}]: len ${chunk.length}, chunk: '${chunk.trim()}'`);
-    });
-
-    const exit = await spawned.onExit;
-    const info: CmdResult = {
-        exit: exit.status,
-        stdout: stdout,
-        stderr: stderr,
+    return {
+        stdin: {
+            write(data) { spawned.stdin.write(data instanceof Buffer ? new Uint8Array(data) : data); },
+            end() { spawned.stdin.end(); },
+        },
+        onStdout(cb) {
+            spawned.stdout.onDidReceiveMessage((d) => {
+                cb(Buffer.from(d));
+            });
+        },
+        onStderr(cb) {
+            spawned.stderr.onDidReceiveMessage((d) => {
+                cb(Buffer.from(d));
+            });
+        },
+        onError(cb) { void spawned.onExit.then(() => { }, (err: unknown) => { cb(err instanceof Error ? err : new Error(String(err))); }); },
+        onExit(cb) { void spawned.onExit.then((e) => { cb(e.status); }); },
+        onClose(cb) {
+            void spawned.onExit.then(
+                () => { cb(); },
+                () => { cb(); },
+            );
+        },
     };
-
-    if (exit.status === 0) {
-        log.info(`RemoteExecCtx[${cmdStr()}]: stdout: ${info.stdout.trim()}`);
-    }
-    else {
-        log.error(`RemoteExecCtx[${cmdStr()}]: ${formatCmdErr(info)}`);
-    }
-
-    return { exit: exit.status, stdout, stderr };
 }

src/common/opts.ts

@@ -1,14 +1,5 @@
-import { LogOutputChannel } from "vscode";
-
 export type Envs = Record<string, string | undefined>;
 
-export interface SpawnOpts {
-    env?: Envs,
-    cwd?: string,
-    stdin?: string | undefined,
-    log: LogOutputChannel,
-}
-
 export interface RunOpts {
     cwd?: string,
     env?: Envs,

src/common/spawn.ts

@@ -1,91 +1,60 @@
-import * as chproc from "node:child_process";
-import { CmdResult, SpawnOpts } from "./opts";
+import { StringDecoder } from "node:string_decoder";
+import { CmdResult, RunOpts } from "./opts";
+import { getLogSink } from "../extension/log";
+import type { SpawnedProcess } from "./ctx/execCtx";
 
 let cmdCount: number = 0;
 
-/* eslint-disable @typescript-eslint/no-confusing-void-expression */
-export function spawn(
-    cmd: string,
-    args: string[],
-    opts: SpawnOpts,
-): Promise<CmdResult> {
-    return new Promise((resolve, _) => {
+export function collectOutput(proc: SpawnedProcess, opts?: RunOpts): Promise<CmdResult> {
+    return new Promise((resolve) => {
         cmdCount += 1;
         const cmdId = cmdCount;
+        const log = getLogSink();
+        const tag = `CMD${String(cmdId).padStart(4, "0")}`;
 
-        const cmdStr = () => `[CMD${String(cmdId).padStart(4, "0")}]:`;
+        let stdout = "";
+        let stderr = "";
+        const stdoutDecoder = new StringDecoder("utf-8");
+        const stderrDecoder = new StringDecoder("utf-8");
 
-        let stdout: string = "";
-        let stderr: string = "";
-
-        // TODO: do we need env without inheriting parent's env?
-        const finalEnv = {
-            ...process.env,
-            ...(opts.env ?? {}),
-            BUILDKIT_PROGRESS: "plain",
-        };
-
-        const proc = chproc.spawn(cmd, args, {
-            cwd: opts.cwd,
-            env: finalEnv,
-            stdio: "pipe",
-        });
-        proc.stdout.setEncoding("utf-8");
-        proc.stderr.setEncoding("utf-8");
-
-        if (opts.stdin !== undefined) {
-            proc.stdin.write(opts.stdin);
+        if (opts?.stdin !== undefined) {
+            proc.stdin.write(Buffer.from(opts.stdin));
             proc.stdin.end();
         }
 
-        proc.on("spawn", () => {
-            const strz_args = args.map(e => `'${e}'`).join(", ");
-            // TODO: log env values as well
-            opts.log.info(`${cmdStr()} Running (spawn) ['${cmd}', ${strz_args}]`);
+        proc.onStdout((d) => {
+            const chunk = stdoutDecoder.write(d);
+            stdout += chunk;
+            log.debug(`${tag} stdout: ${chunk.trimEnd()}`);
         });
 
-        proc.on("error", (err: NodeJS.ErrnoException) => {
-            const msg = `${err.code} :: ${err.message} :: :: ${err.syscall}`;
-            const res: CmdResult = {
-                exit: err.errno ?? 255,
-                stdout: "",
-                stderr: err.message,
-            };
-
-            opts.log.error(cmdStr(), msg);
-            // TODO: reject?
-            return resolve(res);
+        proc.onStderr((d) => {
+            const chunk = stderrDecoder.write(d);
+            stderr += chunk;
+            log.debug(`${tag} stderr: ${chunk.trimEnd()}`);
         });
 
-        proc.stdout.on("data", (data: string) => {
-            stdout += data;
-            opts.log.info(cmdStr(), data);
+        proc.onError((err) => {
+            log.error(`${tag} error: ${err.message}`);
+            resolve({ exit: 255, stdout: "", stderr: err.message });
         });
 
-        proc.stderr.on("data", (data: string) => {
-            stderr += data;
-            opts.log.error(cmdStr(), data);
-        });
-
-        proc.on("exit", (code, signal) => {
-            const res: CmdResult = {
-                exit: code ?? (signal ?? 256),
-                stdout: stdout,
-                stderr: stderr,
-            };
+        let exitCode: number = 256;
 
-            if (code !== 0) {
-                opts.log.error(`${cmdStr()} Command failed with {code / signal ${res.exit}}`);
-            }
-            else {
-                opts.log.info(`${cmdStr()} command exit: ${res.exit}`);
+        proc.onExit((code) => {
+            exitCode = code ?? 256;
+            if (exitCode !== 0) {
+                log.error(`${tag} exited ${exitCode}`);
             }
+        });
 
-            return resolve(res);
+        proc.onClose(() => {
+            stdout += stdoutDecoder.end();
+            stderr += stderrDecoder.end();
+            resolve({ exit: exitCode, stdout, stderr });
         });
     });
 }
-/* eslint-enable @typescript-eslint/no-confusing-void-expression */
 
 export function formatCmdErr(res: CmdResult): string {
     return `Error: ${res.exit}: stdout: [${res.stdout.trim()}] stderr: [${res.stderr.trim()}]`;