fcvm-metal-arm: a second public IPv4 address, to switch over to

.github/workflows/lambda-tests.yml

@@ -61,6 +61,9 @@ jobs:
       - name: Dev-host credential boundaries
         run: python3 -S -B scripts/test-dev-credential-boundary.py
 
+      - name: Metal secondary IPv4 persists across boots without restarting networking
+        run: python3 -S -B scripts/test-fcvm-secondary-ipv4.py
+
       - name: Jumpbox native Claude installation boundaries
         run: python3 -S -B scripts/test-jumpbox-native-claude.py
 

README.md

@@ -394,6 +394,12 @@ ssh nextjs
 ssh io
 ```
 
+The ARM host's second inbound address is reported by
+`firecracker_dev_public_ip_next`. Terraform persists its secondary private
+address in netplan and adds it live without restarting networking. The original
+address remains the outbound address until a separate cutover.
+
+
 The aliases use Elastic IPs for the three long-lived `us-west-1` boxes and fixed private
 address `172.31.48.10` for `io-box`. Dev servers carry a dedicated dev-hop key, not the
 `fcvm-ec2` admin key, so ordinary dev-to-dev access does not grant an admin shell on a

fcvm-dns-test.tf

@@ -36,7 +36,7 @@ locals {
 
   fcvm_dns_test_client_cidrs = concat(
     [for s in local.dev_fleet_subnets : s.ipv6_cidr_block],
-    [for e in aws_eip.firecracker_dev : "${e.public_ip}/32"],
+    [for ip in local.firecracker_dev_public_ips : "${ip}/32"],
     [for e in aws_eip.x86_dev : "${e.public_ip}/32"],
   )
 }

firecracker-dev.tf

@@ -128,6 +128,8 @@ resource "aws_network_interface" "firecracker_dev" {
   subnet_id         = local.subnet_ids_by_az[var.firecracker_availability_zone]
   security_groups   = [aws_security_group.firecracker_dev[0].id]
   ipv6_prefix_count = 1
+  # One secondary private IPv4 address, for the second public address (aws_eip.firecracker_dev_next).
+  private_ips_count = 1
 
   # The host also gets its own IPv6 address, not just the /80 its VMs route, but CreateNetworkInterface
   # takes only one of the four IPv6 parameters (address count, addresses, prefix count, prefixes), so
@@ -301,6 +303,60 @@ resource "aws_eip_association" "firecracker_dev" {
   allocation_id = aws_eip.firecracker_dev[0].id
 }
 
+# A second public address, to move the box to (owner, 2026-10-25: "add a second ip to this box and we will
+# switch over"). It maps to the ENI's secondary private address, so inbound traffic reaches the box on either
+# address while outbound traffic still leaves from the primary private address, through aws_eip.firecracker_dev.
+# Every allowlist of the box's address (fcvm-dns-test.tf, gpu-box.tf, ohio.tf, runner-vpc.tf) admits both. The switch is a
+# later change: associate this address with the primary private address and release the old one.
+resource "aws_eip" "firecracker_dev_next" {
+  count  = var.enable_firecracker_instance ? 1 : 0
+  domain = "vpc"
+
+  tags = {
+    Name = "fcvm-metal-arm-eip-next"
+  }
+}
+
+resource "aws_eip_association" "firecracker_dev_next" {
+  count                = var.enable_firecracker_instance ? 1 : 0
+  allocation_id        = aws_eip.firecracker_dev_next[0].id
+  network_interface_id = aws_network_interface.firecracker_dev[0].id
+  private_ip_address   = one(setsubtract(aws_network_interface.firecracker_dev[0].private_ips, [aws_network_interface.firecracker_dev[0].private_ip]))
+}
+
+# Persist the address in netplan for ordinary reboots and add it live without reconfiguring
+# the primary interface. State Manager also restores it if the managed file is removed.
+resource "aws_ssm_association" "firecracker_dev_secondary_ipv4" {
+  count = var.enable_firecracker_instance ? 1 : 0
+
+  name                             = "AWS-RunShellScript"
+  association_name                 = "fcvm-arm-secondary-ipv4"
+  schedule_expression              = "rate(1 day)"
+  wait_for_success_timeout_seconds = 300
+
+  targets {
+    key    = "InstanceIds"
+    values = [aws_instance.firecracker_dev[0].id]
+  }
+
+  parameters = {
+    commands         = <<-EOT
+      python3 - '${one(setsubtract(aws_network_interface.firecracker_dev[0].private_ips, [aws_network_interface.firecracker_dev[0].private_ip]))}' <<'PYSECONDARY'
+      ${indent(6, chomp(file("${path.module}/scripts/fcvm-secondary-ipv4.py")))}
+      PYSECONDARY
+    EOT
+    executionTimeout = "120"
+  }
+}
+
+locals {
+  # Both public addresses of fcvm-metal-arm, current first. Allowlists of the box use this list.
+  firecracker_dev_public_ips = concat(
+    [for e in aws_eip.firecracker_dev : e.public_ip],
+    [for e in aws_eip.firecracker_dev_next : e.public_ip],
+  )
+}
+
 # Output the instance ID and connection command
 output "firecracker_dev_instance_id" {
   description = "Instance ID of Firecracker dev instance"
@@ -317,6 +373,11 @@ output "firecracker_dev_public_ip" {
   value       = var.enable_firecracker_instance ? aws_eip.firecracker_dev[0].public_ip : null
 }
 
+output "firecracker_dev_public_ip_next" {
+  description = "Second public IP of Firecracker dev instance (Elastic IP on the secondary private address)"
+  value       = var.enable_firecracker_instance ? aws_eip.firecracker_dev_next[0].public_ip : null
+}
+
 output "firecracker_dev_ssh_command" {
   description = "Command to connect to Firecracker dev instance via SSH"
   value       = var.enable_firecracker_instance ? "ssh -i ~/.ssh/${var.firecracker_key_name} ubuntu@${aws_eip.firecracker_dev[0].public_ip}" : null

gpu-box.tf

@@ -113,7 +113,7 @@ resource "aws_security_group" "gpu_box" {
   dynamic "ingress" {
     for_each = merge(
       { for i, e in aws_eip.nextjs_dev : "nextjs-dev" => e.public_ip },
-      { for i, e in aws_eip.firecracker_dev : "fcvm-metal-arm" => e.public_ip },
+      { for i, ip in local.firecracker_dev_public_ips : "fcvm-metal-arm-${i}" => ip },
       { for i, e in aws_eip.x86_dev : "fcvm-metal-x86" => e.public_ip },
     )
     content {

ohio.tf

@@ -102,6 +102,7 @@ resource "aws_security_group" "ohio_runner" {
     cidr_blocks = [
       "${aws_eip.jumpbox[0].public_ip}/32",
       "${aws_eip.firecracker_dev[0].public_ip}/32",
+      "${aws_eip.firecracker_dev_next[0].public_ip}/32",
       "${aws_eip.x86_dev[0].public_ip}/32",
     ]
     description = "SSH from fcvm runners (self) + operator EIPs (jumpbox, dev servers); SSM elsewhere"

runner-vpc.tf

@@ -140,9 +140,10 @@ resource "aws_security_group" "runner" {
     protocol  = "tcp"
     self      = true # fcvm runner-to-runner only
     cidr_blocks = [
-      "${aws_eip.jumpbox[0].public_ip}/32",         # jumpbox (management host)
-      "${aws_eip.firecracker_dev[0].public_ip}/32", # fcvm-metal-arm dev server
-      "${aws_eip.x86_dev[0].public_ip}/32",         # fcvm-metal-x86 dev server
+      "${aws_eip.jumpbox[0].public_ip}/32",              # jumpbox (management host)
+      "${aws_eip.firecracker_dev[0].public_ip}/32",      # fcvm-metal-arm dev server
+      "${aws_eip.firecracker_dev_next[0].public_ip}/32", # fcvm-metal-arm, second address
+      "${aws_eip.x86_dev[0].public_ip}/32",              # fcvm-metal-x86 dev server
     ]
     description = "SSH from fcvm runners (self) + operator EIPs (jumpbox, dev servers); SSM elsewhere"
   }

scripts/fcvm-secondary-ipv4.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""Persist the metal host's Terraform-assigned secondary address without a network restart."""
+import ipaddress
+import json
+from pathlib import Path
+import re
+import subprocess
+import sys
+
+
+CONFIG = Path('/etc/netplan/61-fcvm-secondary-ipv4.yaml')
+
+
+def configure(address, config=CONFIG, run=subprocess.run):
+    address = str(ipaddress.IPv4Address(address))
+    routes = json.loads(run(['ip', '-j', '-4', 'route', 'show', 'default'],
+                            check=True, capture_output=True, text=True).stdout)
+    if not routes:
+        raise RuntimeError('no primary IPv4 route')
+    interface = min(routes, key=lambda route: route.get('metric', 0))['dev']
+    if not re.fullmatch(r'[A-Za-z0-9_.:-]+', interface):
+        raise ValueError('invalid primary interface')
+    # /32 adds the host address without replacing its DHCP subnet or default route.
+    wanted = (f'network:\n  version: 2\n  ethernets:\n    {interface}:\n'
+              f'      addresses:\n        - {address}/32\n')
+    previous = config.read_text() if config.exists() else None
+    if previous != wanted:
+        config.parent.mkdir(parents=True, exist_ok=True)
+        temporary = config.with_suffix('.yaml.tmp')
+        temporary.write_text(wanted)
+        temporary.chmod(0o600)
+        temporary.replace(config)
+    try:
+        # Generate boot configuration; applying netplan live would disrupt SSH and VMs.
+        run(['netplan', 'generate'], check=True)
+    except subprocess.CalledProcessError:
+        if previous is None:
+            config.unlink(missing_ok=True)
+        else:
+            config.write_text(previous)
+            config.chmod(0o600)
+        run(['netplan', 'generate'], check=True)
+        raise
+    run(['ip', '-4', 'address', 'replace', f'{address}/32', 'dev', interface], check=True)
+
+
+if __name__ == '__main__':
+    configure(sys.argv[1])

scripts/test-fcvm-dns-test.py

@@ -39,7 +39,7 @@ def test_client_ips_come_from_terraform_not_literals(self):
         self.assertIn("request_ip = { in = local.fcvm_dns_test_client_cidrs }", token)
         cidrs = TF.split("fcvm_dns_test_client_cidrs = concat(", 1)[1].split("\n  )", 1)[0]
         self.assertIn("[for s in local.dev_fleet_subnets : s.ipv6_cidr_block]", cidrs)
-        self.assertIn('[for e in aws_eip.firecracker_dev : "${e.public_ip}/32"]', cidrs)
+        self.assertIn('[for ip in local.firecracker_dev_public_ips : "${ip}/32"]', cidrs)
         self.assertIn('[for e in aws_eip.x86_dev : "${e.public_ip}/32"]', cidrs)
         self.assertNotRegex(TF, r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", "no IPv4 literal")
         self.assertNotRegex(TF, r'"[0-9a-f]{1,4}:[0-9a-f:]+(/\d+)?"', "no IPv6 literal")

scripts/test-fcvm-secondary-ipv4.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+"""The added address survives boot and never restarts the live primary interface."""
+import importlib.util
+import json
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+
+ROOT = Path(__file__).resolve().parent.parent
+spec = importlib.util.spec_from_file_location('secondary_ipv4', ROOT / 'scripts/fcvm-secondary-ipv4.py')
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+
+
+class SecondaryAddress(unittest.TestCase):
+    def test_persists_and_applies_without_replacing_dhcp_or_routes(self):
+        with tempfile.TemporaryDirectory() as directory:
+            config = Path(directory) / '61-fcvm-secondary-ipv4.yaml'
+            calls = []
+            def run(args, **kwargs):
+                calls.append(args)
+                return subprocess.CompletedProcess(args, 0, json.dumps([{'dev': 'ens5', 'metric': 100}]))
+            module.configure('10.0.1.42', config, run)
+            first = config.read_bytes()
+            self.assertIn(b'ens5:\n      addresses:\n        - 10.0.1.42/32\n', first)
+            self.assertNotIn(b'dhcp4:', first)
+            self.assertEqual(config.stat().st_mode & 0o777, 0o600)
+            self.assertIn(['netplan', 'generate'], calls)
+            self.assertEqual(calls[-1], ['ip', '-4', 'address', 'replace', '10.0.1.42/32', 'dev', 'ens5'])
+            module.configure('10.0.1.42', config, run)
+            self.assertEqual(config.read_bytes(), first)
+            self.assertNotIn(['netplan', 'apply'], calls)
+            self.assertFalse(any('restart' in call for call in calls))
+
+    def test_failed_boot_configuration_rolls_back_before_any_live_change(self):
+        with tempfile.TemporaryDirectory() as directory:
+            config = Path(directory) / '61-fcvm-secondary-ipv4.yaml'
+            original = 'network:\n  version: 2\n'
+            config.write_text(original)
+            calls = []
+            def run(args, **kwargs):
+                calls.append(args)
+                if args == ['netplan', 'generate'] and calls.count(args) == 1:
+                    raise subprocess.CalledProcessError(1, args)
+                return subprocess.CompletedProcess(args, 0, '[{"dev":"ens5"}]')
+            with self.assertRaises(subprocess.CalledProcessError):
+                module.configure('10.0.1.42', config, run)
+            self.assertEqual(config.read_text(), original)
+            self.assertFalse(any('replace' in call for call in calls))
+
+    def test_no_address_can_be_written_without_an_interface_or_valid_ip(self):
+        with tempfile.TemporaryDirectory() as directory:
+            config = Path(directory) / 'netplan.yaml'
+            def run(args, **kwargs):
+                return subprocess.CompletedProcess(args, 0, '[]')
+            with self.assertRaises(RuntimeError):
+                module.configure('10.0.1.42', config, run)
+            with self.assertRaises(ValueError):
+                module.configure('10.0.1.42; reboot', config, run)
+            self.assertFalse(config.exists())
+
+    def test_managed_association_runs_the_script_on_the_arm_host(self):
+        tf = (ROOT / 'firecracker-dev.tf').read_text()
+        self.assertIn('resource "aws_ssm_association" "firecracker_dev_secondary_ipv4"', tf)
+        association = tf.split('resource "aws_ssm_association" "firecracker_dev_secondary_ipv4"', 1)[1].split('\n}\n', 1)[0]
+        self.assertIn('file("${path.module}/scripts/fcvm-secondary-ipv4.py")', association)
+        self.assertIn('aws_instance.firecracker_dev[0].id', association)
+        self.assertIn('wait_for_success_timeout_seconds', association)
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=True)

scripts/test-gpu-box.py

@@ -131,7 +131,8 @@ def test_ssh_only_from_the_granted_boxes_and_no_fleet_egress(self):
         ingress, egress = sg.split('dynamic "egress"')
         self.assertNotIn('0.0.0.0/0', ingress)
         # exactly the boxes holding the grant, each a /32 of its own Elastic IP
-        self.assertEqual(re.findall(r'aws_eip\.(\w+) :', ingress), ['nextjs_dev', 'firecracker_dev', 'x86_dev'])
+        self.assertEqual(re.findall(r'aws_eip\.(\w+) :', ingress), ['nextjs_dev', 'x86_dev'])
+        self.assertIn('for i, ip in local.firecracker_dev_public_ips', ingress)
         self.assertIn('cidr_blocks = ["${ingress.value}/32"]', ingress)
         # the group's description is immutable (a change replaces the group under any running gbox)
         self.assertIn('description = "On-demand GPU test box: SSH from nextjs-dev only"', sg)

scripts/test-ohio-runner-prep.py

@@ -134,7 +134,7 @@ def rules(text, name):
         app = (ROOT / "runner-app.tf").read_text()
         self.assertEqual(rules(TF, "ohio_runner_app")[1], 0, "app runners run other people's code: no inbound")
         self.assertEqual(rules(TF, "ohio_runner_app"), rules(app, "runner_app"))
-        ops = ["aws_eip.jumpbox[0]", "aws_eip.firecracker_dev[0]", "aws_eip.x86_dev[0]"]
+        ops = ["aws_eip.jumpbox[0]", "aws_eip.firecracker_dev[0]", "aws_eip.firecracker_dev_next[0]", "aws_eip.x86_dev[0]"]
         for op in ops:
             self.assertIn(op, re.search(r'resource "aws_security_group" "ohio_runner" \{.*?^\}', TF, re.S | re.M).group())
 

scripts/test-runner-iam-boundary.py

@@ -212,7 +212,7 @@ def test_fcvm_runner_ssh_is_self_and_operator_eips_only(self):
         self.assertNotIn('ipv6_cidr_blocks', rule)
         cidrs = re.search(r'cidr_blocks\s*=\s*\[\n(.*?)\n\s*\]', rule, re.S).group(1)
         self.assertEqual(re.findall(r'aws_eip\.(\w+)\[0\]\.public_ip', cidrs),
-                         ['jumpbox', 'firecracker_dev', 'x86_dev'])
+                         ['jumpbox', 'firecracker_dev', 'firecracker_dev_next', 'x86_dev'])
         self.assertNotRegex(cidrs, r'"\d+\.\d+\.\d+\.\d+/\d+"')