From c27ca7d134437ee84a1b4ea976fc235868dafbc8 Mon Sep 17 00:00:00 2001 From: ejc3 Date: Sun, 25 Oct 2026 10:10:45 +0000 Subject: [PATCH 1/3] fcvm-metal-arm: a second public IPv4 address, to switch over to - firecracker-dev.tf: one secondary private IPv4 address on the box's ENI (private_ips_count = 1), a new Elastic IP (firecracker_dev_next) associated with it, local.firecracker_dev_public_ips holding both public addresses, and an output for the new one - fcvm-dns-test.tf, gpu-box.tf, ohio.tf, runner-vpc.tf: admit both addresses wherever the box's address is allowlisted - scripts/test-*.py: the three tests that pin those allowlists name the new address Outbound traffic still leaves through the current address. The switch is a later change: associate the new address with the primary private address and release the old one. Tested: terraform validate (1.10.3); test-fcvm-dns-test.py, test-runner-iam-boundary.py, test-ohio-runner-prep.py, test-dev-credential-boundary.py --- fcvm-dns-test.tf | 2 +- firecracker-dev.tf | 36 +++++++++++++++++++++++++++++ gpu-box.tf | 2 +- ohio.tf | 1 + runner-vpc.tf | 7 +++--- scripts/test-fcvm-dns-test.py | 2 +- scripts/test-ohio-runner-prep.py | 2 +- scripts/test-runner-iam-boundary.py | 2 +- 8 files changed, 46 insertions(+), 8 deletions(-) diff --git a/fcvm-dns-test.tf b/fcvm-dns-test.tf index 0f78437e..01452dd1 100644 --- a/fcvm-dns-test.tf +++ b/fcvm-dns-test.tf @@ -36,7 +36,7 @@ locals { fcvm_dns_test_client_cidrs = concat( [for s in local.dev_fleet_subnets : s.ipv6_cidr_block], - [for e in aws_eip.firecracker_dev : "${e.public_ip}/32"], + [for ip in local.firecracker_dev_public_ips : "${ip}/32"], [for e in aws_eip.x86_dev : "${e.public_ip}/32"], ) } diff --git a/firecracker-dev.tf b/firecracker-dev.tf index d17da91d..98dee0ee 100644 --- a/firecracker-dev.tf +++ b/firecracker-dev.tf @@ -128,6 +128,8 @@ resource "aws_network_interface" "firecracker_dev" { subnet_id = local.subnet_ids_by_az[var.firecracker_availability_zone] security_groups = [aws_security_group.firecracker_dev[0].id] ipv6_prefix_count = 1 + # One secondary private IPv4 address, for the second public address (aws_eip.firecracker_dev_next). + private_ips_count = 1 # The host also gets its own IPv6 address, not just the /80 its VMs route, but CreateNetworkInterface # takes only one of the four IPv6 parameters (address count, addresses, prefix count, prefixes), so @@ -301,6 +303,35 @@ resource "aws_eip_association" "firecracker_dev" { allocation_id = aws_eip.firecracker_dev[0].id } +# A second public address, to move the box to (owner, 2026-10-25: "add a second ip to this box and we will +# switch over"). It maps to the ENI's secondary private address, so inbound traffic reaches the box on either +# address while outbound traffic still leaves from the primary private address, through aws_eip.firecracker_dev. +# Every allowlist of the box's address (fcvm-dns-test.tf, gpu-box.tf, ohio.tf, runner-vpc.tf) admits both. The switch is a +# later change: associate this address with the primary private address and release the old one. +resource "aws_eip" "firecracker_dev_next" { + count = var.enable_firecracker_instance ? 1 : 0 + domain = "vpc" + + tags = { + Name = "fcvm-metal-arm-eip-next" + } +} + +resource "aws_eip_association" "firecracker_dev_next" { + count = var.enable_firecracker_instance ? 1 : 0 + allocation_id = aws_eip.firecracker_dev_next[0].id + network_interface_id = aws_network_interface.firecracker_dev[0].id + private_ip_address = one(setsubtract(aws_network_interface.firecracker_dev[0].private_ips, [aws_network_interface.firecracker_dev[0].private_ip])) +} + +locals { + # Both public addresses of fcvm-metal-arm, current first. Allowlists of the box use this list. + firecracker_dev_public_ips = concat( + [for e in aws_eip.firecracker_dev : e.public_ip], + [for e in aws_eip.firecracker_dev_next : e.public_ip], + ) +} + # Output the instance ID and connection command output "firecracker_dev_instance_id" { description = "Instance ID of Firecracker dev instance" @@ -317,6 +348,11 @@ output "firecracker_dev_public_ip" { value = var.enable_firecracker_instance ? aws_eip.firecracker_dev[0].public_ip : null } +output "firecracker_dev_public_ip_next" { + description = "Second public IP of Firecracker dev instance (Elastic IP on the secondary private address)" + value = var.enable_firecracker_instance ? aws_eip.firecracker_dev_next[0].public_ip : null +} + output "firecracker_dev_ssh_command" { description = "Command to connect to Firecracker dev instance via SSH" value = var.enable_firecracker_instance ? "ssh -i ~/.ssh/${var.firecracker_key_name} ubuntu@${aws_eip.firecracker_dev[0].public_ip}" : null diff --git a/gpu-box.tf b/gpu-box.tf index aa5dff1e..4ee11a7b 100644 --- a/gpu-box.tf +++ b/gpu-box.tf @@ -113,7 +113,7 @@ resource "aws_security_group" "gpu_box" { dynamic "ingress" { for_each = merge( { for i, e in aws_eip.nextjs_dev : "nextjs-dev" => e.public_ip }, - { for i, e in aws_eip.firecracker_dev : "fcvm-metal-arm" => e.public_ip }, + { for i, ip in local.firecracker_dev_public_ips : "fcvm-metal-arm-${i}" => ip }, { for i, e in aws_eip.x86_dev : "fcvm-metal-x86" => e.public_ip }, ) content { diff --git a/ohio.tf b/ohio.tf index e2161c20..365b8f34 100644 --- a/ohio.tf +++ b/ohio.tf @@ -102,6 +102,7 @@ resource "aws_security_group" "ohio_runner" { cidr_blocks = [ "${aws_eip.jumpbox[0].public_ip}/32", "${aws_eip.firecracker_dev[0].public_ip}/32", + "${aws_eip.firecracker_dev_next[0].public_ip}/32", "${aws_eip.x86_dev[0].public_ip}/32", ] description = "SSH from fcvm runners (self) + operator EIPs (jumpbox, dev servers); SSM elsewhere" diff --git a/runner-vpc.tf b/runner-vpc.tf index 713bc111..6c8c13b6 100644 --- a/runner-vpc.tf +++ b/runner-vpc.tf @@ -140,9 +140,10 @@ resource "aws_security_group" "runner" { protocol = "tcp" self = true # fcvm runner-to-runner only cidr_blocks = [ - "${aws_eip.jumpbox[0].public_ip}/32", # jumpbox (management host) - "${aws_eip.firecracker_dev[0].public_ip}/32", # fcvm-metal-arm dev server - "${aws_eip.x86_dev[0].public_ip}/32", # fcvm-metal-x86 dev server + "${aws_eip.jumpbox[0].public_ip}/32", # jumpbox (management host) + "${aws_eip.firecracker_dev[0].public_ip}/32", # fcvm-metal-arm dev server + "${aws_eip.firecracker_dev_next[0].public_ip}/32", # fcvm-metal-arm, second address + "${aws_eip.x86_dev[0].public_ip}/32", # fcvm-metal-x86 dev server ] description = "SSH from fcvm runners (self) + operator EIPs (jumpbox, dev servers); SSM elsewhere" } diff --git a/scripts/test-fcvm-dns-test.py b/scripts/test-fcvm-dns-test.py index 901fee71..c405ee94 100644 --- a/scripts/test-fcvm-dns-test.py +++ b/scripts/test-fcvm-dns-test.py @@ -39,7 +39,7 @@ def test_client_ips_come_from_terraform_not_literals(self): self.assertIn("request_ip = { in = local.fcvm_dns_test_client_cidrs }", token) cidrs = TF.split("fcvm_dns_test_client_cidrs = concat(", 1)[1].split("\n )", 1)[0] self.assertIn("[for s in local.dev_fleet_subnets : s.ipv6_cidr_block]", cidrs) - self.assertIn('[for e in aws_eip.firecracker_dev : "${e.public_ip}/32"]', cidrs) + self.assertIn('[for ip in local.firecracker_dev_public_ips : "${ip}/32"]', cidrs) self.assertIn('[for e in aws_eip.x86_dev : "${e.public_ip}/32"]', cidrs) self.assertNotRegex(TF, r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", "no IPv4 literal") self.assertNotRegex(TF, r'"[0-9a-f]{1,4}:[0-9a-f:]+(/\d+)?"', "no IPv6 literal") diff --git a/scripts/test-ohio-runner-prep.py b/scripts/test-ohio-runner-prep.py index 807354de..6b915530 100644 --- a/scripts/test-ohio-runner-prep.py +++ b/scripts/test-ohio-runner-prep.py @@ -134,7 +134,7 @@ def rules(text, name): app = (ROOT / "runner-app.tf").read_text() self.assertEqual(rules(TF, "ohio_runner_app")[1], 0, "app runners run other people's code: no inbound") self.assertEqual(rules(TF, "ohio_runner_app"), rules(app, "runner_app")) - ops = ["aws_eip.jumpbox[0]", "aws_eip.firecracker_dev[0]", "aws_eip.x86_dev[0]"] + ops = ["aws_eip.jumpbox[0]", "aws_eip.firecracker_dev[0]", "aws_eip.firecracker_dev_next[0]", "aws_eip.x86_dev[0]"] for op in ops: self.assertIn(op, re.search(r'resource "aws_security_group" "ohio_runner" \{.*?^\}', TF, re.S | re.M).group()) diff --git a/scripts/test-runner-iam-boundary.py b/scripts/test-runner-iam-boundary.py index 95ab2974..fd61efa3 100644 --- a/scripts/test-runner-iam-boundary.py +++ b/scripts/test-runner-iam-boundary.py @@ -212,7 +212,7 @@ def test_fcvm_runner_ssh_is_self_and_operator_eips_only(self): self.assertNotIn('ipv6_cidr_blocks', rule) cidrs = re.search(r'cidr_blocks\s*=\s*\[\n(.*?)\n\s*\]', rule, re.S).group(1) self.assertEqual(re.findall(r'aws_eip\.(\w+)\[0\]\.public_ip', cidrs), - ['jumpbox', 'firecracker_dev', 'x86_dev']) + ['jumpbox', 'firecracker_dev', 'firecracker_dev_next', 'x86_dev']) self.assertNotRegex(cidrs, r'"\d+\.\d+\.\d+\.\d+/\d+"') From 4a9c2e8d36ff8453fac0d1ea961753cfaa324e69 Mon Sep 17 00:00:00 2001 From: ejc3 Date: Sun, 25 Oct 2026 12:20:53 +0000 Subject: [PATCH 2/3] Persist the secondary IPv4 address and validate both SSH allowlists --- .github/workflows/lambda-tests.yml | 3 ++ README.md | 6 +++ firecracker-dev.tf | 25 ++++++++++ scripts/fcvm-secondary-ipv4.py | 48 +++++++++++++++++++ scripts/test-fcvm-secondary-ipv4.py | 73 +++++++++++++++++++++++++++++ scripts/test-gpu-box.py | 3 +- 6 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 scripts/fcvm-secondary-ipv4.py create mode 100644 scripts/test-fcvm-secondary-ipv4.py diff --git a/.github/workflows/lambda-tests.yml b/.github/workflows/lambda-tests.yml index da384054..25b19e96 100644 --- a/.github/workflows/lambda-tests.yml +++ b/.github/workflows/lambda-tests.yml @@ -61,6 +61,9 @@ jobs: - name: Dev-host credential boundaries run: python3 -S -B scripts/test-dev-credential-boundary.py + - name: Metal secondary IPv4 persists across boots without restarting networking + run: python3 -S -B scripts/test-fcvm-secondary-ipv4.py + - name: Jumpbox native Claude installation boundaries run: python3 -S -B scripts/test-jumpbox-native-claude.py diff --git a/README.md b/README.md index 006ebf7b..c0b4880a 100644 --- a/README.md +++ b/README.md @@ -394,6 +394,12 @@ ssh nextjs ssh io ``` +The ARM host's second inbound address is reported by +`firecracker_dev_public_ip_next`. Terraform persists its secondary private +address in netplan and adds it live without restarting networking. The original +address remains the outbound address until a separate cutover. + + The aliases use Elastic IPs for the three long-lived `us-west-1` boxes and fixed private address `172.31.48.10` for `io-box`. Dev servers carry a dedicated dev-hop key, not the `fcvm-ec2` admin key, so ordinary dev-to-dev access does not grant an admin shell on a diff --git a/firecracker-dev.tf b/firecracker-dev.tf index 98dee0ee..87c8aac7 100644 --- a/firecracker-dev.tf +++ b/firecracker-dev.tf @@ -324,6 +324,31 @@ resource "aws_eip_association" "firecracker_dev_next" { private_ip_address = one(setsubtract(aws_network_interface.firecracker_dev[0].private_ips, [aws_network_interface.firecracker_dev[0].private_ip])) } +# Persist the address in netplan for ordinary reboots and add it live without reconfiguring +# the primary interface. State Manager also restores it if the managed file is removed. +resource "aws_ssm_association" "firecracker_dev_secondary_ipv4" { + count = var.enable_firecracker_instance ? 1 : 0 + + name = "AWS-RunShellScript" + association_name = "fcvm-arm-secondary-ipv4" + schedule_expression = "rate(1 day)" + wait_for_success_timeout = "5m" + + targets { + key = "InstanceIds" + values = [aws_instance.firecracker_dev[0].id] + } + + parameters = { + commands = <<-EOT + python3 - '${one(setsubtract(aws_network_interface.firecracker_dev[0].private_ips, [aws_network_interface.firecracker_dev[0].private_ip]))}' <<'PYSECONDARY' + ${indent(6, chomp(file("${path.module}/scripts/fcvm-secondary-ipv4.py")))} + PYSECONDARY + EOT + executionTimeout = "120" + } +} + locals { # Both public addresses of fcvm-metal-arm, current first. Allowlists of the box use this list. firecracker_dev_public_ips = concat( diff --git a/scripts/fcvm-secondary-ipv4.py b/scripts/fcvm-secondary-ipv4.py new file mode 100644 index 00000000..9be3daf6 --- /dev/null +++ b/scripts/fcvm-secondary-ipv4.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Persist the metal host's Terraform-assigned secondary address without a network restart.""" +import ipaddress +import json +from pathlib import Path +import re +import subprocess +import sys + + +CONFIG = Path('/etc/netplan/61-fcvm-secondary-ipv4.yaml') + + +def configure(address, config=CONFIG, run=subprocess.run): + address = str(ipaddress.IPv4Address(address)) + routes = json.loads(run(['ip', '-j', '-4', 'route', 'show', 'default'], + check=True, capture_output=True, text=True).stdout) + if not routes: + raise RuntimeError('no primary IPv4 route') + interface = min(routes, key=lambda route: route.get('metric', 0))['dev'] + if not re.fullmatch(r'[A-Za-z0-9_.:-]+', interface): + raise ValueError('invalid primary interface') + # /32 adds the host address without replacing its DHCP subnet or default route. + wanted = (f'network:\n version: 2\n ethernets:\n {interface}:\n' + f' addresses:\n - {address}/32\n') + previous = config.read_text() if config.exists() else None + if previous != wanted: + config.parent.mkdir(parents=True, exist_ok=True) + temporary = config.with_suffix('.yaml.tmp') + temporary.write_text(wanted) + temporary.chmod(0o600) + temporary.replace(config) + try: + # Generate boot configuration; applying netplan live would disrupt SSH and VMs. + run(['netplan', 'generate'], check=True) + except subprocess.CalledProcessError: + if previous is None: + config.unlink(missing_ok=True) + else: + config.write_text(previous) + config.chmod(0o600) + run(['netplan', 'generate'], check=True) + raise + run(['ip', '-4', 'address', 'replace', f'{address}/32', 'dev', interface], check=True) + + +if __name__ == '__main__': + configure(sys.argv[1]) diff --git a/scripts/test-fcvm-secondary-ipv4.py b/scripts/test-fcvm-secondary-ipv4.py new file mode 100644 index 00000000..2c31ef9b --- /dev/null +++ b/scripts/test-fcvm-secondary-ipv4.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""The added address survives boot and never restarts the live primary interface.""" +import importlib.util +import json +from pathlib import Path +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parent.parent +spec = importlib.util.spec_from_file_location('secondary_ipv4', ROOT / 'scripts/fcvm-secondary-ipv4.py') +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class SecondaryAddress(unittest.TestCase): + def test_persists_and_applies_without_replacing_dhcp_or_routes(self): + with tempfile.TemporaryDirectory() as directory: + config = Path(directory) / '61-fcvm-secondary-ipv4.yaml' + calls = [] + def run(args, **kwargs): + calls.append(args) + return subprocess.CompletedProcess(args, 0, json.dumps([{'dev': 'ens5', 'metric': 100}])) + module.configure('10.0.1.42', config, run) + first = config.read_bytes() + self.assertIn(b'ens5:\n addresses:\n - 10.0.1.42/32\n', first) + self.assertNotIn(b'dhcp4:', first) + self.assertEqual(config.stat().st_mode & 0o777, 0o600) + self.assertIn(['netplan', 'generate'], calls) + self.assertEqual(calls[-1], ['ip', '-4', 'address', 'replace', '10.0.1.42/32', 'dev', 'ens5']) + module.configure('10.0.1.42', config, run) + self.assertEqual(config.read_bytes(), first) + self.assertNotIn(['netplan', 'apply'], calls) + self.assertFalse(any('restart' in call for call in calls)) + + def test_failed_boot_configuration_rolls_back_before_any_live_change(self): + with tempfile.TemporaryDirectory() as directory: + config = Path(directory) / '61-fcvm-secondary-ipv4.yaml' + original = 'network:\n version: 2\n' + config.write_text(original) + calls = [] + def run(args, **kwargs): + calls.append(args) + if args == ['netplan', 'generate'] and calls.count(args) == 1: + raise subprocess.CalledProcessError(1, args) + return subprocess.CompletedProcess(args, 0, '[{"dev":"ens5"}]') + with self.assertRaises(subprocess.CalledProcessError): + module.configure('10.0.1.42', config, run) + self.assertEqual(config.read_text(), original) + self.assertFalse(any('replace' in call for call in calls)) + + def test_no_address_can_be_written_without_an_interface_or_valid_ip(self): + with tempfile.TemporaryDirectory() as directory: + config = Path(directory) / 'netplan.yaml' + def run(args, **kwargs): + return subprocess.CompletedProcess(args, 0, '[]') + with self.assertRaises(RuntimeError): + module.configure('10.0.1.42', config, run) + with self.assertRaises(ValueError): + module.configure('10.0.1.42; reboot', config, run) + self.assertFalse(config.exists()) + + def test_managed_association_runs_the_script_on_the_arm_host(self): + tf = (ROOT / 'firecracker-dev.tf').read_text() + self.assertIn('resource "aws_ssm_association" "firecracker_dev_secondary_ipv4"', tf) + association = tf.split('resource "aws_ssm_association" "firecracker_dev_secondary_ipv4"', 1)[1].split('\n}\n', 1)[0] + self.assertIn('file("${path.module}/scripts/fcvm-secondary-ipv4.py")', association) + self.assertIn('aws_instance.firecracker_dev[0].id', association) + self.assertIn('wait_for_success_timeout', association) + + +if __name__ == '__main__': + unittest.main(buffer=True) diff --git a/scripts/test-gpu-box.py b/scripts/test-gpu-box.py index 71de8f34..2a3a6fbd 100755 --- a/scripts/test-gpu-box.py +++ b/scripts/test-gpu-box.py @@ -131,7 +131,8 @@ def test_ssh_only_from_the_granted_boxes_and_no_fleet_egress(self): ingress, egress = sg.split('dynamic "egress"') self.assertNotIn('0.0.0.0/0', ingress) # exactly the boxes holding the grant, each a /32 of its own Elastic IP - self.assertEqual(re.findall(r'aws_eip\.(\w+) :', ingress), ['nextjs_dev', 'firecracker_dev', 'x86_dev']) + self.assertEqual(re.findall(r'aws_eip\.(\w+) :', ingress), ['nextjs_dev', 'x86_dev']) + self.assertIn('for i, ip in local.firecracker_dev_public_ips', ingress) self.assertIn('cidr_blocks = ["${ingress.value}/32"]', ingress) # the group's description is immutable (a change replaces the group under any running gbox) self.assertIn('description = "On-demand GPU test box: SSH from nextjs-dev only"', sg) From c7a1b8cfe15292cf46d2b8fbb8f3b68027d2b9e9 Mon Sep 17 00:00:00 2001 From: ejc3 Date: Sun, 25 Oct 2026 12:34:38 +0000 Subject: [PATCH 3/3] Use the provider timeout field in seconds --- firecracker-dev.tf | 8 ++++---- scripts/test-fcvm-secondary-ipv4.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/firecracker-dev.tf b/firecracker-dev.tf index 87c8aac7..770e2811 100644 --- a/firecracker-dev.tf +++ b/firecracker-dev.tf @@ -329,10 +329,10 @@ resource "aws_eip_association" "firecracker_dev_next" { resource "aws_ssm_association" "firecracker_dev_secondary_ipv4" { count = var.enable_firecracker_instance ? 1 : 0 - name = "AWS-RunShellScript" - association_name = "fcvm-arm-secondary-ipv4" - schedule_expression = "rate(1 day)" - wait_for_success_timeout = "5m" + name = "AWS-RunShellScript" + association_name = "fcvm-arm-secondary-ipv4" + schedule_expression = "rate(1 day)" + wait_for_success_timeout_seconds = 300 targets { key = "InstanceIds" diff --git a/scripts/test-fcvm-secondary-ipv4.py b/scripts/test-fcvm-secondary-ipv4.py index 2c31ef9b..c82c521d 100644 --- a/scripts/test-fcvm-secondary-ipv4.py +++ b/scripts/test-fcvm-secondary-ipv4.py @@ -66,7 +66,7 @@ def test_managed_association_runs_the_script_on_the_arm_host(self): association = tf.split('resource "aws_ssm_association" "firecracker_dev_secondary_ipv4"', 1)[1].split('\n}\n', 1)[0] self.assertIn('file("${path.module}/scripts/fcvm-secondary-ipv4.py")', association) self.assertIn('aws_instance.firecracker_dev[0].id', association) - self.assertIn('wait_for_success_timeout', association) + self.assertIn('wait_for_success_timeout_seconds', association) if __name__ == '__main__':