From ad467aadb2202731cc3c198039f0a68ce0bfe86b Mon Sep 17 00:00:00 2001 From: ejc3 Date: Sun, 25 Oct 2026 12:30:53 +0000 Subject: [PATCH] fcvm fleet: an aggregated-streaming switch, off (ejc3/fcvm#1232) local.fcvm_fleet_aggregated (false) gates everything the aggregated streams need, so with it off the agent arguments, the coordinator command, its task definition and both security groups are what they were: - Hosts: --aggregate-port 20300 (private: layout, stats; not admitted) and --aggregate-tls-port, the port after the slots' (20208), which the fleet security group's IPv4 range then includes. Over IPv6 a host aggregator has its own address, ::1, on 443, already admitted. - Coordinator: --ec2-aggregate-listen on loopback and --ec2-aggregate-tls with its certificate in /run/fcvm-fleet/tls; the task gets a volume there (the root file system is read-only), net.ipv4.ip_unprivileged_port_start=443 (the image runs as 65534), 443 in its security group over IPv4 and IPv6, and the account's dualStackIPv6 ECS setting so the task has an IPv6 address. The coordinator already holds ec2:Describe*, which reading its own interface's public IPv4 address needs. scripts/test-fcvm-fleet.py: the coordinator security group admits nothing but the gated 443 rule, the slot range ends at the aggregator port only with the switch on, and AggregatedTests pins the switch off and every addition gated by it. --- fcvm-fleet.tf | 63 +++++++++++++++++++++++++++++++++++--- scripts/test-fcvm-fleet.py | 42 +++++++++++++++++++++++-- 2 files changed, 98 insertions(+), 7 deletions(-) diff --git a/fcvm-fleet.tf b/fcvm-fleet.tf index e456912..b376a21 100644 --- a/fcvm-fleet.tf +++ b/fcvm-fleet.tf @@ -78,14 +78,25 @@ locals { # The agent's --tls-publish-base and the fleet security group's port range both come from these two numbers. fcvm_fleet_slot_tls_port_base = 20200 fcvm_fleet_host_gpu_slots = 8 # a g4dn.metal's T4s + + # Aggregated streaming (ejc3/fcvm#1232): each GPU host's aggregator, and the EC2 half's beside the coordinator, serve + # browsers with TLS under per-launch names (agg--, agg-ec2-). Off: the page runs direct (one stream + # per tile). On needs a host AMI and a coordinator image with --aggregate-tls-port and --ec2-aggregate-tls (fcvm #1232's + # stack); an image without them refuses the argument and does not start, so the switch moves with the images. + fcvm_fleet_aggregated = false + # A host aggregator's IPv4 path: the port after the slots', on the host's public IPv4 address (one address per host, a + # port per stream). Over IPv6 it has its own address, ::1, on 443. Its private listener (layout, stats) is + # on fcvm_fleet_aggregate_port, which the security group does not admit. + fcvm_fleet_aggregate_tls_port = local.fcvm_fleet_slot_tls_port_base + local.fcvm_fleet_host_gpu_slots + fcvm_fleet_aggregate_port = 20300 fcvm_fleet_slot_tls_ports = { from = local.fcvm_fleet_slot_tls_port_base - to = local.fcvm_fleet_slot_tls_port_base + local.fcvm_fleet_host_gpu_slots - 1 + to = local.fcvm_fleet_aggregated ? local.fcvm_fleet_aggregate_tls_port : local.fcvm_fleet_slot_tls_port_base + local.fcvm_fleet_host_gpu_slots - 1 } # The zone's authoritative nameservers: the coordinator publishes a launch's names only once both answer with them. fcvm_dev_nameservers = ["cullen.ns.cloudflare.com", "lilyana.ns.cloudflare.com"] - fcvm_fleet_host_agent_args = join(" ", [ + fcvm_fleet_host_agent_args = join(" ", concat([ "--dial wss://${local.fcvm_dev_agent_path}", "--access-secret ${aws_secretsmanager_secret.fcvm_fleet_agent_access.name}", "--tls-secret ${aws_secretsmanager_secret.fcvm_fleet_tls.name}", @@ -100,7 +111,10 @@ locals { "--public-ipv4-from-imds", "--spot-notice-url http://169.254.169.254", "--store-ready-file /var/lib/fcvm-fleet/store-ready", - ]) + ], local.fcvm_fleet_aggregated ? [ + "--aggregate-port ${local.fcvm_fleet_aggregate_port}", + "--aggregate-tls-port ${local.fcvm_fleet_aggregate_tls_port}", + ] : [])) # The coordinator's arguments (fcvm fleet/PROTOCOL.md, "Coordinator", "Running it"); its state bucket and key come from # the task's environment. It owns the host groups' desired capacity: 0 after 10 minutes with no viewer, 1 for the next @@ -114,6 +128,8 @@ locals { ) : [], ["--ec2", "aws", "--ec2-tiles", "8", "--ec2-regions", "us-east-1", "--ec2-template", "fcvm-fleet-tile-x86_64", "--ec2-on-demand-template", "fcvm-fleet-tile-on-demand-x86_64"], ["--dns-zone", "fcvm.dev", "--dns-token-secret", aws_secretsmanager_secret.fcvm_fleet_dns_token.name, "--dns-zone-id", var.fcvm_dev_zone_id, "--tls-secret", aws_secretsmanager_secret.fcvm_fleet_tls.name, "--secrets-region", local.fcvm_fleet_home_region, "--dns-nameservers", join(",", local.fcvm_dev_nameservers)], + # The EC2 half's aggregator beside the coordinator: its layout on loopback, browsers on 443 of the task's own addresses. + local.fcvm_fleet_aggregated ? ["--ec2-aggregate-listen", "127.0.0.1:${local.fcvm_fleet_aggregate_port}", "--ec2-aggregate-tls", "--ec2-aggregate-tls-dir", "/run/fcvm-fleet/tls"] : [], ) fcvm_fleet_coordinator_running = local.fcvm_fleet_coordinator_enabled && local.fcvm_fleet_coordinator_tag != "" @@ -653,6 +669,20 @@ resource "aws_security_group" "fcvm_fleet_coordinator" { description = "fcvm fleet coordinator task: nothing in (browsers and agents arrive through its Cloudflare tunnel), all out" vpc_id = module.fcvm_fleet_east2.vpc_id + # With aggregated streaming, browsers read the EC2 half's aggregator on 443 of the task's own addresses, not through the + # tunnel (which Access gates and which would put Cloudflare in the media path). The coordinator's own port stays closed. + dynamic "ingress" { + for_each = local.fcvm_fleet_aggregated ? [1] : [] + content { + description = "EC2 half aggregator, TLS" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + } + egress { description = "All outbound" from_port = 0 @@ -685,8 +715,15 @@ resource "aws_ecs_task_definition" "fcvm_fleet_coordinator" { cpu_architecture = "ARM64" } + dynamic "volume" { + for_each = local.fcvm_fleet_aggregated ? ["fleet-run"] : [] + content { + name = volume.value + } + } + container_definitions = jsonencode([ - { + merge({ name = "coordinator" image = local.fcvm_fleet_coordinator_running ? nonsensitive(data.aws_ssm_parameter.fcvm_coordinator_release.value) : "public.ecr.aws/docker/library/busybox:stable" essential = true @@ -704,7 +741,14 @@ resource "aws_ecs_task_definition" "fcvm_fleet_coordinator" { logDriver = "awslogs" options = { awslogs-group = aws_cloudwatch_log_group.fcvm_fleet_coordinator.name, awslogs-region = local.fcvm_fleet_home_region, awslogs-stream-prefix = "coordinator" } } - }, + }, + # The EC2 half's aggregator: the image runs as 65534 and binds 443 (ip_unprivileged_port_start covers IPv6 too), + # and its certificate file goes on a task volume (the root file system is read-only). Absent when off, so the + # task definition is the same as before. + [for _ in(local.fcvm_fleet_aggregated ? [1] : []) : { + systemControls = [{ namespace = "net.ipv4.ip_unprivileged_port_start", value = "443" }] + mountPoints = [{ sourceVolume = "fleet-run", containerPath = "/run/fcvm-fleet", readOnly = false }] + }]...), { name = "cloudflared" image = "cloudflare/cloudflared:${local.cloudflared_version}" # the pin claw-master-tunnel.tf keeps @@ -729,6 +773,15 @@ resource "aws_ecs_task_definition" "fcvm_fleet_coordinator" { } } +# A Fargate task gets an IPv6 address in a dual-stack subnet only with this account setting; the EC2 half's aggregator is +# named at it (agg-ec2- AAAA). Without it the aggregator has its A record only. +resource "aws_ecs_account_setting_default" "fcvm_fleet_dual_stack" { + count = local.fcvm_fleet_aggregated ? 1 : 0 + provider = aws.fcvm_fleet + name = "dualStackIPv6" + value = "enabled" +} + resource "aws_ecs_service" "fcvm_fleet_coordinator" { depends_on = [ephemeral.aws_ssm_parameter.fcvm_coordinator_release_guard] provider = aws.fcvm_fleet diff --git a/scripts/test-fcvm-fleet.py b/scripts/test-fcvm-fleet.py index c140ef1..dc303bb 100644 --- a/scripts/test-fcvm-fleet.py +++ b/scripts/test-fcvm-fleet.py @@ -26,8 +26,13 @@ def test_the_security_group_admits_tcp_443_and_the_slot_ports_and_nothing_else(s self.assertIn("to_port = 443", sg) self.assertIn('protocol = "tcp"', sg) self.assertNotIn("22", sg) + # The coordinator task admits nothing, except 443 for the EC2 half's aggregator with aggregated streaming on + # (AggregatedTests). coord = block('resource "aws_security_group" "fcvm_fleet_coordinator"') - self.assertNotIn("ingress", coord) + self.assertEqual(coord.count("ingress"), 1, coord) + self.assertIn("for_each = local.fcvm_fleet_aggregated ? [1] : []", coord) + self.assertIn("from_port = 443", coord) + self.assertIn("to_port = 443", coord) def test_every_az_gets_a_public_ipv6_subnet_and_flow_logs(self): sub = block('resource "aws_subnet" "public"', MOD) @@ -325,7 +330,10 @@ def test_the_slot_ports_are_admitted_over_ipv4_and_nothing_wider(self): slots = int(re.search(r"fcvm_fleet_host_gpu_slots\s+= (\d+)", TF).group(1)) self.assertEqual((base, slots), (20200, 8)) self.assertIn("from = local.fcvm_fleet_slot_tls_port_base", TF) - self.assertIn("to = local.fcvm_fleet_slot_tls_port_base + local.fcvm_fleet_host_gpu_slots - 1", TF) + # The slots' ports, and with aggregated streaming the host aggregator's port after them. + self.assertIn( + "to = local.fcvm_fleet_aggregated ? local.fcvm_fleet_aggregate_tls_port : " + "local.fcvm_fleet_slot_tls_port_base + local.fcvm_fleet_host_gpu_slots - 1", TF) # The agent forwards from the same base. self.assertIn('"--tls-publish-base ${local.fcvm_fleet_slot_tls_port_base}"', TF) self.assertIn('"--public-ipv4-from-imds"', TF) @@ -345,5 +353,35 @@ def test_the_coordinator_confirms_names_at_the_zones_nameservers(self): self.assertIn('"--dns-nameservers", join(",", local.fcvm_dev_nameservers)', cmd) + +class AggregatedTests(unittest.TestCase): + """Aggregated streaming (ejc3/fcvm#1232) is one switch, off, and off changes nothing staging runs.""" + + def test_the_switch_is_off(self): + self.assertRegex(TF, r"\n fcvm_fleet_aggregated = false\n") + + def test_everything_it_adds_is_gated_by_it(self): + agent = TF.split("fcvm_fleet_host_agent_args = join(", 1)[1].split("\n\n", 1)[0] + self.assertIn('], local.fcvm_fleet_aggregated ? [', agent) + self.assertIn('"--aggregate-tls-port ${local.fcvm_fleet_aggregate_tls_port}"', agent) + cmd = TF.split("fcvm_fleet_coordinator_command = concat(", 1)[1].split("\n )\n", 1)[0] + self.assertIn('local.fcvm_fleet_aggregated ? ["--ec2-aggregate-listen", "127.0.0.1:', cmd) + self.assertIn('"--ec2-aggregate-tls"', cmd) + task = block('resource "aws_ecs_task_definition" "fcvm_fleet_coordinator"') + self.assertIn('for_each = local.fcvm_fleet_aggregated ? ["fleet-run"] : []', task) + self.assertIn("[for _ in(local.fcvm_fleet_aggregated ? [1] : []) : {", task) + self.assertIn('namespace = "net.ipv4.ip_unprivileged_port_start", value = "443"', task) + self.assertIn('containerPath = "/run/fcvm-fleet"', task) + self.assertIn('"/run/fcvm-fleet/tls"', cmd) + dual = block('resource "aws_ecs_account_setting_default" "fcvm_fleet_dual_stack"') + self.assertIn("count = local.fcvm_fleet_aggregated ? 1 : 0", dual) + + def test_the_host_aggregator_port_follows_the_slots(self): + self.assertIn( + "fcvm_fleet_aggregate_tls_port = local.fcvm_fleet_slot_tls_port_base + local.fcvm_fleet_host_gpu_slots", TF) + priv = int(re.search(r"fcvm_fleet_aggregate_port\s+= (\d+)", TF).group(1)) + # The private listener is outside the admitted range. + self.assertFalse(20200 <= priv <= 20208, priv) + if __name__ == "__main__": unittest.main(verbosity=2, buffer=True)