Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,16 @@ All notable changes to KTMS are listed here. The format follows

## [Unreleased]

### Security
- cloudflared 2026.10.3 or newer for the node's built-in tunnel: older releases trust the eMudhra
intermediates whose keys leaked in October 2026. `install-node.sh` replaces an older one on a
re-run, without `--upgrade-deps`, and restarts the node onto it; `ktms-node doctor` reports it
as a problem and the agent warns about it at start.

### Fixed
- `install-node.sh --upgrade-deps` restarts the node after updating cloudflared. The running tunnel
kept the old binary until the next restart.

## [1.3.4] - 2026-10-26

### Fixed
Expand Down
5 changes: 4 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,7 @@ so the node must be reachable; the agent can do that itself:
```bash
export NODE_KEY=... # or --node-key-file
./bin/ktms-node --proxy-addr proxy.example.com:443 --tls \
--engine ollama --expose=cloudflared # needs the cloudflared binary
--engine ollama --expose=cloudflared # needs cloudflared 2026.10.3 or newer
```

With `--expose=cloudflared` the agent starts a localhost gateway in front of the engine and a
Expand All @@ -325,6 +325,9 @@ access key (sent to the proxy as the node auth token, never printed) and forward
`POST /v1/chat/completions` and `GET /v1/models`; pulling or deleting models is not reachable.
Slow responses are kept alive with padding so the tunnel's 100 s idle cutoff does not drop them.
Quick tunnels have no SLA; for a stable address use `--expose=none` and your own `--endpoint`.
cloudflared older than 2026.10.3 trusts the eMudhra intermediates whose keys leaked in October
2026: the agent warns about it at start and `ktms-node doctor` reports it; re-running
`install-node.sh` updates it.

Flags: `--proxy-addr` (default `127.0.0.1:50051`), `--tls` (dial over TLS with the system CA
roots, for a proxy behind a TLS load balancer; off by default), `--node-id` (defaults to the
Expand Down
48 changes: 48 additions & 0 deletions internal/nodeagent/cloudflared.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package nodeagent

import (
"context"
"os/exec"
"regexp"
"strconv"
"strings"
"time"
)

// MinCloudflared is the oldest cloudflared that no longer trusts the eMudhra
// intermediates whose keys leaked in October 2026.
const MinCloudflared = "2026.10.3"

var cloudflaredVersionRe = regexp.MustCompile(`version (\d+\.\d+\.\d+)`)

// cloudflaredVersion pulls the version out of `cloudflared --version`; "" when
// it has none, as in development builds.
func cloudflaredVersion(out string) string {
if m := cloudflaredVersionRe.FindStringSubmatch(out); m != nil {
return m[1]
}
return ""
}

// cloudflaredTooOld reports whether a known version is below MinCloudflared.
func cloudflaredTooOld(v string) bool {
if v == "" {
return false
}
have, want := strings.Split(v, "."), strings.Split(MinCloudflared, ".")
for i := range want {
h, _ := strconv.Atoi(have[i])
w, _ := strconv.Atoi(want[i])
if h != w {
return h < w
}
}
return false
}

func runCloudflaredVersion(ctx context.Context, bin string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
out, err := exec.CommandContext(ctx, bin, "--version").CombinedOutput()
return string(out), err
}
19 changes: 14 additions & 5 deletions internal/nodeagent/doctor.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ type Doctor struct {
DialTCP func(ctx context.Context, addr string) error
UsesProxy func(addr string) bool // true when HTTPS_PROXY or similar routes addr; gRPC honors it, a raw dial does not
LookPath func(name string) (string, error)
RunCmd func(ctx context.Context, name string, args ...string) error
RunCmd func(ctx context.Context, name string, args ...string) (string, error) // combined output
MachOArchs func(path string) ([]string, error)
Now func() time.Time
GOOS string
Expand Down Expand Up @@ -86,10 +86,11 @@ func (d *Doctor) defaults() {
d.LookPath = exec.LookPath
}
if d.RunCmd == nil {
d.RunCmd = func(ctx context.Context, name string, args ...string) error {
d.RunCmd = func(ctx context.Context, name string, args ...string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
return exec.CommandContext(ctx, name, args...).Run()
out, err := exec.CommandContext(ctx, name, args...).CombinedOutput()
return string(out), err
}
}
if d.MachOArchs == nil {
Expand Down Expand Up @@ -505,11 +506,19 @@ func (d *Doctor) checkTunnel(ctx context.Context) []Finding {
}
bin = p
}
if err := d.RunCmd(ctx, bin, "--version"); err != nil {
out, err := d.RunCmd(ctx, bin, "--version")
if err != nil {
return []Finding{{"Tunnel", LevelProblem, fmt.Sprintf("cloudflared (%s) is there but does not start (%s).", bin, shortErr(err)), "On macOS it may be blocked by Gatekeeper: re-download it with curl or brew instead of a browser."}}
}
var findings []Finding
if v := cloudflaredVersion(out); cloudflaredTooOld(v) {
findings = append(findings, Finding{"Tunnel", LevelProblem, fmt.Sprintf("cloudflared %s (%s) still trusts the leaked eMudhra certificates; %s or newer is needed.", v, bin, MinCloudflared), "Re-run install-node.sh (it updates cloudflared and restarts the node), or brew upgrade cloudflared."})
}
if err := d.DialTCP(ctx, "region1.v2.argotunnel.com:7844"); err != nil {
return []Finding{{"Tunnel", LevelProblem, "The tunnel cannot come up: this network blocks outbound port 7844 to Cloudflare.", "Try another network, or ask IT to allow outbound TCP/UDP 7844. (" + shortErr(err) + ")"}}
return append(findings, Finding{"Tunnel", LevelProblem, "The tunnel cannot come up: this network blocks outbound port 7844 to Cloudflare.", "Try another network, or ask IT to allow outbound TCP/UDP 7844. (" + shortErr(err) + ")"})
}
if findings != nil {
return findings
}
return []Finding{{"Tunnel", LevelOK, "cloudflared starts and Cloudflare is reachable.", ""}}
}
Expand Down
34 changes: 32 additions & 2 deletions internal/nodeagent/doctor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,9 @@ func TestDoctorTunnel(t *testing.T) {
mk := func() *Doctor {
d := newDoctor(AgentConfig{Expose: ExposeCloudflared})
d.LookPath = func(string) (string, error) { return "/usr/local/bin/cloudflared", nil }
d.RunCmd = func(context.Context, string, ...string) error { return nil }
d.RunCmd = func(context.Context, string, ...string) (string, error) {
return "cloudflared version 2026.10.4 (built 2026-10-26)", nil
}
d.DialTCP = func(context.Context, string) error { return nil }
return d
}
Expand All @@ -336,7 +338,7 @@ func TestDoctorTunnel(t *testing.T) {
}

d = mk()
d.RunCmd = func(context.Context, string, ...string) error { return errors.New("killed: 9") }
d.RunCmd = func(context.Context, string, ...string) (string, error) { return "", errors.New("killed: 9") }
if f := one(t, d.checkTunnel(ctx), "Tunnel"); f.Level != LevelProblem || !strings.Contains(f.Fix, "Gatekeeper") {
t.Fatalf("blocked binary: %+v", f)
}
Expand All @@ -351,6 +353,34 @@ func TestDoctorTunnel(t *testing.T) {
if f := one(t, d.checkTunnel(ctx), "Tunnel"); f.Level != LevelProblem || !strings.Contains(f.Message, "7844") {
t.Fatalf("blocked port: %+v", f)
}

d = mk()
d.RunCmd = func(context.Context, string, ...string) (string, error) {
return "cloudflared version 2026.10.1 (built 2026-10-15-17:08 UTC)", nil
}
if f := one(t, d.checkTunnel(ctx), "Tunnel"); f.Level != LevelProblem || !strings.Contains(f.Message, "eMudhra") || !strings.Contains(f.Fix, "install-node.sh") {
t.Fatalf("old cloudflared: %+v", f)
}

d = mk()
d.RunCmd = func(context.Context, string, ...string) (string, error) { return "cloudflared version DEV", nil }
if f := one(t, d.checkTunnel(ctx), "Tunnel"); f.Level != LevelOK {
t.Fatalf("a build without a version is not flagged: %+v", f)
}
}

func TestCloudflaredTooOld(t *testing.T) {
for v, want := range map[string]bool{
"2026.10.1": true, "2026.10.2": true, "2026.9.12": true, "2025.12.0": true,
"2026.10.3": false, "2026.10.4": false, "2026.11.0": false, "2027.1.0": false, "": false,
} {
if got := cloudflaredTooOld(v); got != want {
t.Errorf("cloudflaredTooOld(%q) = %v, want %v", v, got, want)
}
}
if v := cloudflaredVersion("cloudflared version 2026.10.4 (built 2026-10-26-11:13 UTC)\n"); v != "2026.10.4" {
t.Errorf("version = %q", v)
}
}

func TestDoctorEndpoint(t *testing.T) {
Expand Down
5 changes: 5 additions & 0 deletions internal/nodeagent/expose.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ func (a *Agent) startExposure(ctx context.Context) (urls <-chan string, stopped
}
bin = p
}
if out, err := runCloudflaredVersion(ctx, bin); err == nil {
if v := cloudflaredVersion(out); cloudflaredTooOld(v) {
log.Warn().Str("cloudflared", v).Str("min", MinCloudflared).Msg("cloudflared trusts the leaked eMudhra certificates: update it (re-run install-node.sh or brew upgrade cloudflared)")
}
}

raw := make([]byte, 32)
if _, err := rand.Read(raw); err != nil {
Expand Down
6 changes: 5 additions & 1 deletion internal/nodeagent/expose_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,16 @@ func (f *fakeRegistry) UnregisterNode(context.Context, *pb.UnregisterNodeRequest
return &pb.UnregisterNodeResponse{Ok: true}, nil
}

// fakeVersion answers the version check the agent runs before the tunnel.
const fakeVersion = `[ "$1" = --version ] && { echo "cloudflared version 2026.10.4 (built 2026-10-26)"; exit 0; }`

// fakeCloudflared prints a different address on each launch; the first run
// exits after a moment, like a dropped tunnel.
func fakeCloudflared(t *testing.T) string {
t.Helper()
dir := t.TempDir()
script := `#!/bin/sh
` + fakeVersion + `
n=$(cat "` + dir + `/runs" 2>/dev/null || echo 0)
n=$((n+1))
echo $n > "` + dir + `/runs"
Expand Down Expand Up @@ -149,7 +153,7 @@ func TestExposeUnknownModeAndMissingBinary(t *testing.T) {
func writeScript(t *testing.T, body string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "cloudflared")
if err := os.WriteFile(path, []byte("#!/bin/sh\n"+body), 0o755); err != nil {
if err := os.WriteFile(path, []byte("#!/bin/sh\n"+fakeVersion+"\n"+body), 0o755); err != nil {
t.Fatal(err)
}
return path
Expand Down
2 changes: 1 addition & 1 deletion internal/nodeagent/registration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ func TestRunStartsNoTunnelWhileRefused(t *testing.T) {
dir := t.TempDir()
marker := filepath.Join(dir, "started")
bin := filepath.Join(dir, "cloudflared")
if err := os.WriteFile(bin, []byte("#!/bin/sh\ntouch "+marker+"\nsleep 30\n"), 0o755); err != nil {
if err := os.WriteFile(bin, []byte("#!/bin/sh\n"+fakeVersion+"\ntouch "+marker+"\nsleep 30\n"), 0o755); err != nil {
t.Fatal(err)
}
f := &scriptedRegistry{alwaysRefuse: denied()} // never runs out, so timing cannot let it through
Expand Down
20 changes: 17 additions & 3 deletions scripts/install-node.sh
Original file line number Diff line number Diff line change
Expand Up @@ -585,10 +585,23 @@ ensure_ollama() {
OLLAMA_CHANGED=1
}

# Older cloudflared trusts the eMudhra intermediates whose keys leaked in October 2026.
CF_MIN=2026.10.3

# cf_older BIN: true when BIN reports a version below $CF_MIN; an unknown version is not.
cf_older() {
v=$("$1" --version 2>/dev/null | sed -n 's/.*version \([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\).*/\1/p' | head -n 1)
[ -n "$v" ] || return 1
[ "$(printf '%s\n%s\n' "$v" "$CF_MIN" | sort -t. -k1,1n -k2,2n -k3,3n | head -n 1)" != "$CF_MIN" ]
}

ensure_cloudflared() {
[ -z "$ENDPOINT" ] || return 0
CF_BIN="$BASE/bin/cloudflared"
if [ -x "$CF_BIN" ] && [ "$UPGRADE" = 0 ]; then return 0; fi
if [ -x "$CF_BIN" ] && [ "$UPGRADE" = 0 ]; then
cf_older "$CF_BIN" || return 0
say "cloudflared is older than $CF_MIN and trusts leaked eMudhra certificates: updating it."
fi
tag=$(latest_tag cloudflare/cloudflared)
if [ "$OS" = darwin ]; then asset="cloudflared-darwin-$ARCH.tgz"; else asset="cloudflared-linux-$ARCH"; fi
step "Installing cloudflared $tag"
Expand All @@ -600,6 +613,7 @@ ensure_cloudflared() {
"$CF_BIN.new" --version >/dev/null 2>&1 || die "the downloaded cloudflared does not run on this machine"
mv "$CF_BIN.new" "$CF_BIN"
rm -rf "$tmp"
CF_CHANGED=1 # the running tunnel still uses the old binary until the node restarts
}

# stage_agent downloads, verifies and smoke-tests the agent, leaving it in $BASE/bin/ktms-node.new.
Expand Down Expand Up @@ -899,7 +913,7 @@ main() {
systemctl --user show-environment >/dev/null 2>&1 || die "systemd user services are not available in this session; log in normally (not via su) and retry"
loginctl enable-linger "$(id -un)" 2>/dev/null || warn "could not enable lingering: the node will stop when you log out (run: sudo loginctl enable-linger $(id -un))"
fi
OLLAMA_CHANGED=0 AGENT_STAGED=0 SCHEDULE_REPLACED=0
OLLAMA_CHANGED=0 CF_CHANGED=0 AGENT_STAGED=0 SCHEDULE_REPLACED=0
import_legacy_key
store_key
stage_agent
Expand All @@ -918,7 +932,7 @@ main() {
ensure_model

step "Starting the node"
node_changed=0
node_changed=$CF_CHANGED
if [ "$AGENT_STAGED" = 1 ]; then
svc_stop node
[ -x "$BASE/bin/ktms-node" ] && cp -p "$BASE/bin/ktms-node" "$BASE/bin/ktms-node.prev"
Expand Down
18 changes: 16 additions & 2 deletions scripts/install-node_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -211,8 +211,8 @@ fake_ollama_bin() {
printf '#!/bin/sh\ncase "$1" in\nserve) exec sleep 300;;\npull) echo "$2" >> "$FAKE_STATE/models";;\n--version) echo 0.0.0;;\nesac\n' >"$1"
chmod +x "$1"
}
fake_cloudflared_bin() {
printf '#!/bin/sh\n[ "$1" = --version ] && echo cloudflared\nexit 0\n' >"$1"
fake_cloudflared_bin() { # path [version]
printf '#!/bin/sh\n[ "$1" = --version ] && echo "cloudflared version %s (built 2099-01-01)"\nexit 0\n' "${2:-2099.1.0}" >"$1"
chmod +x "$1"
}

Expand Down Expand Up @@ -359,6 +359,20 @@ eq "linux: re-run exits 0" 0 "$RC"
eq "linux: re-run touches no running service" "$BEFORE" "$(count_calls 'restart|stop')"
has "linux: re-run says agent up to date" "already installed" "$OUT"

fake_cloudflared_bin "$BASE/bin/cloudflared" 2026.10.1
NODE_BEFORE=$(count_calls 'restart ktms-node')
OUT=$(run_install $ARGS --max-concurrency 2 --context 4096)
RC=$?
eq "linux: re-run with an old cloudflared exits 0" 0 "$RC"
has "linux: old cloudflared explained" "older than 2026.10.3" "$OUT"
has "linux: old cloudflared replaced without --upgrade-deps" "version 2099.1.0" "$("$BASE/bin/cloudflared" --version)"
eq "linux: node restarted onto the new cloudflared" "$((NODE_BEFORE + 1))" "$(count_calls 'restart ktms-node')"
fake_cloudflared_bin "$BASE/bin/cloudflared" 2026.10.3
NODE_BEFORE=$(count_calls 'restart ktms-node')
OUT=$(run_install $ARGS --max-concurrency 2 --context 4096)
hasnt "linux: cloudflared at the minimum is kept" "older than" "$OUT"
eq "linux: kept cloudflared restarts nothing" "$NODE_BEFORE" "$(count_calls 'restart ktms-node')"

CALLS=$(count_calls 'restart|stop')
OUT=$(run_install $ARGS --max-concurrency 2 --context 4096 --doctor)
RC=$?
Expand Down
Loading