From 10b0cd91797c7bec4a70dd32a8ec8cdbaa9f54ec Mon Sep 17 00:00:00 2001 From: ejc3 Date: Sun, 25 Oct 2026 14:01:59 +0000 Subject: [PATCH 1/5] fcvm-fleet: a pool of stopped EC2 tiles the coordinator starts at a wake The coordinator in ejc3/fcvm#1273 keeps its EC2 tiles stopped between wakes (--ec2-pool) and starts them at the next one instead of launching: on g4dn.xlarge from the tile AMI a stopped tile reached its first frame in 15.5 s (persistent spot) against 81 to 84 s for a launch (2026-10-25). g4dn has no hibernation and Auto Scaling warm pools take no spot. - modules/fcvm-fleet-region: launch template fcvm-fleet-tile-pool-, the spot tile with a persistent request that stops on interruption, its spot request tagged Purpose=fcvm-fleet-tile; in tile_template_arns, so RunInstances may use it. - The coordinator's role: StartInstances/StopInstances and CreateTags/DeleteTags (keys FleetPool and FleetTile only) on instances tagged Purpose=fcvm-fleet-tile, and CancelSpotInstanceRequests on requests so tagged (a persistent request launches a replacement when its instance is terminated, so the coordinator cancels it first). - Coordinator arguments --ec2-pool 8 --ec2-pool-template fcvm-fleet-tile-pool-x86_64 behind local.fcvm_fleet_tile_pool. Standing cost: 8 stopped 32 GB gp3 root volumes, $20.48 a month; no Elastic IPs. Tested: python3 scripts/test-fcvm-fleet.py (39 tests OK, 3 new), terraform fmt. terraform validate was not run here: this box has 1.16.6 and main.tf pins 1.10.3. --- fcvm-fleet.tf | 34 ++++++++++++++++++ modules/fcvm-fleet-region/main.tf | 57 +++++++++++++++++++++++++++++-- scripts/test-fcvm-fleet.py | 29 ++++++++++++++++ 3 files changed, 118 insertions(+), 2 deletions(-) diff --git a/fcvm-fleet.tf b/fcvm-fleet.tf index b376a211..18a5c77b 100644 --- a/fcvm-fleet.tf +++ b/fcvm-fleet.tf @@ -53,6 +53,11 @@ locals { fcvm_fleet_coordinator_tag = "bdaa0003" # Most on-demand GPU hosts per region (the fallback group's max_size; owner, 2026-10-24: "Up to 8 on-demand max"). fcvm_fleet_on_demand_max = 8 + # Whether the coordinator keeps its 8 EC2 tiles stopped between wakes and starts them at the next one (--ec2-pool, + # ejc3/fcvm#1273) instead of launching them: a tile then reaches its first frame in about 16 s instead of 81 to 84 s. + # Needs a coordinator image with --ec2-pool (an unknown flag stops the task). Standing cost: 8 stopped 32 GB gp3 root + # volumes, $20.48 a month. + fcvm_fleet_tile_pool = true # Whether the coordinator gives a short spot group's capacity to the region's on-demand group (--on-demand-group, # --spot-wait, ejc3/fcvm#1215). On with the image at c93a01d9 (ejc3/fcvm main, which has those flags); the image at # 8c96e28e refuses an unknown argument and would not start, so the two change together. @@ -127,6 +132,7 @@ locals { ["--spot-wait", "120"], ) : [], ["--ec2", "aws", "--ec2-tiles", "8", "--ec2-regions", "us-east-1", "--ec2-template", "fcvm-fleet-tile-x86_64", "--ec2-on-demand-template", "fcvm-fleet-tile-on-demand-x86_64"], + local.fcvm_fleet_tile_pool ? ["--ec2-pool", "8", "--ec2-pool-template", "fcvm-fleet-tile-pool-x86_64"] : [], ["--dns-zone", "fcvm.dev", "--dns-token-secret", aws_secretsmanager_secret.fcvm_fleet_dns_token.name, "--dns-zone-id", var.fcvm_dev_zone_id, "--tls-secret", aws_secretsmanager_secret.fcvm_fleet_tls.name, "--secrets-region", local.fcvm_fleet_home_region, "--dns-nameservers", join(",", local.fcvm_dev_nameservers)], # The EC2 half's aggregator beside the coordinator: its layout on loopback, browsers on 443 of the task's own addresses. local.fcvm_fleet_aggregated ? ["--ec2-aggregate-listen", "127.0.0.1:${local.fcvm_fleet_aggregate_port}", "--ec2-aggregate-tls", "--ec2-aggregate-tls-dir", "/run/fcvm-fleet/tls"] : [], @@ -646,6 +652,34 @@ resource "aws_iam_role_policy" "fcvm_fleet_coordinator_task" { Resource = "arn:aws:ec2:*:*:instance/*" Condition = { StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } } }, + { + # The tile pool (ejc3/fcvm#1273): stop a tile into the pool when the fleet sleeps and start it at the next wake. + Sid = "StartAndStopTilesOnly" + Effect = "Allow" + Action = ["ec2:StartInstances", "ec2:StopInstances"] + Resource = "arn:aws:ec2:*:*:instance/*" + Condition = { StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } } + }, + { + # Moving a tile into and out of the pool retags it (FleetPool, FleetTile) and nothing else. + Sid = "RetagTilesForThePool" + Effect = "Allow" + Action = ["ec2:CreateTags", "ec2:DeleteTags"] + Resource = "arn:aws:ec2:*:*:instance/*" + Condition = { + StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } + "ForAllValues:StringEquals" = { "aws:TagKeys" = ["FleetPool", "FleetTile"] } + } + }, + { + # A pool tile's spot request is persistent: it is cancelled before its instance is terminated, or it launches a + # replacement. The pool template tags its requests Purpose=fcvm-fleet-tile. + Sid = "CancelTileSpotRequestsOnly" + Effect = "Allow" + Action = "ec2:CancelSpotInstanceRequests" + Resource = "arn:aws:ec2:*:*:spot-instances-request/*" + Condition = { StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } } + }, { Sid = "DescribeEc2" Effect = "Allow" diff --git a/modules/fcvm-fleet-region/main.tf b/modules/fcvm-fleet-region/main.tf index 0f1e82c3..7af732ca 100644 --- a/modules/fcvm-fleet-region/main.tf +++ b/modules/fcvm-fleet-region/main.tf @@ -502,6 +502,59 @@ resource "aws_launch_template" "tile_on_demand" { tags = { Name = "fcvm-fleet-tile-on-demand-${each.key}" } } +# The tile the coordinator keeps a pool of (fcvm fleet coord --ec2-pool --ec2-pool-template +# fcvm-fleet-tile-pool-, ejc3/fcvm#1273): persistent spot that stops on interruption, so the coordinator can stop a +# tile when the fleet sleeps and start it at the next wake (15.5 s to a first frame, against 81 to 84 s for a launch, +# g4dn.xlarge, 2026-10-25). The spot request is tagged too, so the coordinator's CancelSpotInstanceRequests (it cancels a +# persistent request before terminating its instance, or the request launches a replacement) is limited to tiles'. +# Everything else is the spot template's. +resource "aws_launch_template" "tile_pool" { + for_each = toset(local.tile_archs) + name = "fcvm-fleet-tile-pool-${each.key}" + description = "fcvm fleet EC2 comparison tile, ${each.key} (${local.tile_types[each.key]}): persistent spot, stopped between wakes" + image_id = var.tile_amis[each.key] + instance_type = local.tile_types[each.key] + + iam_instance_profile { + name = var.tile_profile_name + } + + instance_market_options { + market_type = "spot" + spot_options { + spot_instance_type = "persistent" + instance_interruption_behavior = "stop" + } + } + + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 1 + } + + network_interfaces { + device_index = 0 + security_groups = [aws_security_group.fleet.id] + associate_public_ip_address = true + ipv6_address_count = 1 + delete_on_termination = true + subnet_id = aws_subnet.public[local.tile_az].id + } + + tag_specifications { + resource_type = "instance" + tags = { Name = "fcvm-fleet-tile", Purpose = "fcvm-fleet-tile" } + } + + tag_specifications { + resource_type = "spot-instances-request" + tags = { Name = "fcvm-fleet-tile", Purpose = "fcvm-fleet-tile" } + } + + tags = { Name = "fcvm-fleet-tile-pool-${each.key}" } +} + output "vpc_id" { value = aws_vpc.this.id } @@ -514,9 +567,9 @@ output "security_group_id" { value = aws_security_group.fleet.id } -# Both markets: the coordinator's RunInstances is limited to these templates. +# Both markets and the pool: the coordinator's RunInstances is limited to these templates. output "tile_template_arns" { - value = concat([for t in aws_launch_template.tile : t.arn], [for t in aws_launch_template.tile_on_demand : t.arn]) + value = concat([for t in aws_launch_template.tile : t.arn], [for t in aws_launch_template.tile_on_demand : t.arn], [for t in aws_launch_template.tile_pool : t.arn]) } output "host_group_desired" { diff --git a/scripts/test-fcvm-fleet.py b/scripts/test-fcvm-fleet.py index dc303bb7..c20131e7 100644 --- a/scripts/test-fcvm-fleet.py +++ b/scripts/test-fcvm-fleet.py @@ -242,6 +242,35 @@ def test_an_on_demand_tile_template_is_a_launchable_template(self): self.assertIn("[for t in aws_launch_template.tile_on_demand : t.arn]", block('output "tile_template_arns"', MOD)) +class TilePool(unittest.TestCase): + TF = CoordinatorLaunchSplit.TF + stmt = CoordinatorLaunchSplit.stmt + + def test_the_pool_template_is_persistent_spot_that_stops_and_tags_its_request(self): + pool = block('resource "aws_launch_template" "tile_pool"', MOD) + self.assertIn('spot_instance_type = "persistent"', pool) + self.assertIn('instance_interruption_behavior = "stop"', pool) + self.assertIn('resource_type = "spot-instances-request"', pool) + self.assertEqual(pool.count('Purpose = "fcvm-fleet-tile"'), 2) + self.assertIn("[for t in aws_launch_template.tile_pool : t.arn]", block('output "tile_template_arns"', MOD)) + + def test_start_stop_retag_and_cancel_are_limited_to_tiles(self): + ss = self.stmt("StartAndStopTilesOnly") + self.assertIn('["ec2:StartInstances", "ec2:StopInstances"]', ss) + self.assertIn('"ec2:ResourceTag/Purpose" = "fcvm-fleet-tile"', ss) + rt = self.stmt("RetagTilesForThePool") + self.assertIn('"ec2:ResourceTag/Purpose" = "fcvm-fleet-tile"', rt) + self.assertIn('"aws:TagKeys" = ["FleetPool", "FleetTile"]', rt) + cs = self.stmt("CancelTileSpotRequestsOnly") + self.assertIn('Resource = "arn:aws:ec2:*:*:spot-instances-request/*"', cs) + self.assertIn('"ec2:ResourceTag/Purpose" = "fcvm-fleet-tile"', cs) + for s in (ss, rt, cs): + self.assertNotIn('Resource = "*"', s) + + def test_the_coordinator_gets_the_pool_flags_only_behind_the_switch(self): + self.assertIn('local.fcvm_fleet_tile_pool ? ["--ec2-pool", "8", "--ec2-pool-template", "fcvm-fleet-tile-pool-x86_64"] : []', self.TF) + + class ModuleDependencies(unittest.TestCase): def test_no_module_level_depends_on(self): # A module-level depends_on defers the module's data sources (the AZ list) to apply time, and the subnets' From 91ae2336653b8e564a7073fe7ee5a2654973735b Mon Sep 17 00:00:00 2001 From: ejc3 Date: Sun, 25 Oct 2026 15:45:42 +0000 Subject: [PATCH 2/5] fcvm-fleet: the coordinator launches the GPU hosts; remove the host auto scaling groups ejc3/fcvm#1257: Auto Scaling took 7 to 13 s after each SetDesiredCapacity to make its first CreateFleet call (CloudTrail, six staging wakes on 2026-10-24 and 25), asked one AZ per call with retries 10 s to 4 minutes apart, and never moved capacity to the on-demand group itself. The coordinator now launches the hosts from the host templates (instant EC2 Fleet over every host subnet, spot first, on demand in the same pass when spot is refused, at most 8 on demand across the fleet) and terminates them while the fleet sleeps. - modules/fcvm-fleet-region: drop aws_autoscaling_group.hosts and hosts_on_demand, the SNS topic, its subscription and the on-demand alarm (it read the on-demand group's metric), and the host_on_demand_max and alert_email variables. New outputs host_template_names (only while hosts are enabled and an AMI exists), host_subnet_ids, host_template_arns, host_types. - fcvm-fleet.tf: the coordinator gets FLEET_HOST_TEMPLATES, FLEET_HOST_SUBNETS and FLEET_HOST_ON_DEMAND_MAX in its environment and --sleep-when-idle in place of --scale-group, --on-demand-group and --spot-wait. Both the running image and the image with the launcher start with these: the old one ignores the environment, the new one has no --scale-group. - IAM: ec2:CreateFleet in the fleet regions; RunInstances from the host templates with the host AMIs, the fleet's subnets and group; a deny on any type outside the four fleet types and on a template launching another kind's types; Purpose fcvm-fleet-tile or fcvm-fleet-host required at launch and for termination; tags at RunInstances or CreateFleet; PassRole for the host role too. The autoscaling statements are gone. - EC2 Fleet's service-linked role (ec2fleet.amazonaws.com), which the account lacks. - scripts/test-fcvm-fleet.py and AGENTS.md follow. Tested: python3 -S -B scripts/test-fcvm-fleet.py (33 tests OK; the 8 new or changed ones fail against main's files); terraform fmt -check; terraform validate of modules/fcvm-fleet-region. --- AGENTS.md | 29 ++-- fcvm-fleet.tf | 182 +++++++++++++++---------- modules/fcvm-fleet-region/main.tf | 212 +++++------------------------- scripts/test-fcvm-fleet.py | 150 ++++++++++----------- 4 files changed, 226 insertions(+), 347 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 368a96fe..d8aac892 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1700,14 +1700,14 @@ trusts anything in `fcvm-fleet`, and no peering, shared subnet or resource polic `fcvm-fleet/agent-access-token`. The apex and `fleet-api.fcvm.dev` go live only on the owner's go-ahead. - **Infrastructure** (`fcvm-fleet.tf`, ejc3/aws#606; `modules/fcvm-fleet-region` once per region): a VPC per region (IPv6 /56, a public subnet per AZ, one security group admitting TCP 443 from anywhere and nothing else, no SSH: SSM only, flow logs - to `fcvm-fleet-flow-logs-`); a spot GPU host group per region over g4dn.metal and g5g.metal (launch template per - architecture, `Ipv6PrefixCount = 1` only, IMDSv2 hop 1); EC2 tile launch templates (g4dn.xlarge, g5g.xlarge) the + to `fcvm-fleet-flow-logs-`); GPU host launch templates per architecture (g4dn.metal, g5g.metal; `Ipv6PrefixCount = 1` + only, IMDSv2 hop 1) the coordinator launches from itself; EC2 tile launch templates (g4dn.xlarge, g5g.xlarge) the coordinator launches itself; the coordinator as one Fargate service (coordinator plus cloudflared) in ECS cluster `fcvm-fleet` in us-east-2 with its state object in S3; the asset bucket; the `fcvm-fleet/tls-wildcard` secret. The Cloudflare-fed secrets (DNS token, tunnel token, agent Access token) come from `fcvm-dev.tf`. - **The two gates are on**: `local.fcvm_fleet_hosts_enabled` (also needing a regional AMI in - `var.fcvm_fleet_host_amis`) and `local.fcvm_fleet_coordinator_enabled`. GPU hosts scale to zero - when idle. Main-only CI builds and tests an ARM64 coordinator image, publishes a full-commit + `var.fcvm_fleet_host_amis`) and `local.fcvm_fleet_coordinator_enabled`. GPU hosts are + terminated when idle. Main-only CI builds and tests an ARM64 coordinator image, publishes a full-commit immutable tag, then invokes `fcvm-coordinator-release`. That function holds the same DynamoDB backend lock as Terraform, clones the running task definition changing only its image, and records the successful digest in `/fcvm-fleet/stage/coordinator-image`. Terraform reads that @@ -1716,17 +1716,16 @@ trusts anything in `fcvm-fleet`, and no peering, shared subnet or resource polic version. An ephemeral apply-time check rejects a saved plan made before that increment, including when a failed release restores the previous image. Infrastructure still changes only through Terraform from a jumpbox. -- **On-demand fallback** (`fcvm-fleet-hosts-on-demand`, beside each spot group): Auto Scaling does not move a spot group's - capacity to on demand by itself, so the coordinator does (`--on-demand-group`, `--spot-wait`): a wake whose spot group - stays short gives the on-demand group the shortfall, and sleep sets both groups to 0. `max_size` is - `local.fcvm_fleet_on_demand_max` = 8 (owner, 2026-10-24: "Up to 8 on-demand max"); the us-east-1 on-demand G and VT - quota (L-DB2E81BA, 384 vCPU) fits 4 g4dn.metal. `local.fcvm_fleet_coordinator_falls_back` passes the flags; it stays - false until the coordinator tag has them. Alarm `fcvm-fleet-on-demand-hosts` (fleet account, the group's region) mails - the owner while any on-demand host is in service. -- The coordinator's task role may launch EC2 only from the tile templates, referencing only the tile AMIs and the fleet's - subnets and group (by exact ARN), only as the two tile types and tagged `Purpose=fcvm-fleet-tile`; terminate only - instances so tagged; pass only the tile role; read and write only `state.json`; read the DNS token and the certificate - and store the certificate it issues. The fleet account has EC2's spot service-linked role. Hosts read the certificate, +- **GPU hosts** (ejc3/fcvm#1257): no auto scaling group. The coordinator launches each host itself, spot first (an instant + EC2 Fleet over the region's host templates and host subnets, `FLEET_HOST_TEMPLATES`, `FLEET_HOST_SUBNETS`), on demand + in the same pass when spot is refused for capacity, at most `local.fcvm_fleet_on_demand_max` = 8 on-demand hosts across + the fleet (owner, 2026-10-24: "Up to 8 on-demand max"; `FLEET_HOST_ON_DEMAND_MAX`), and terminates every host while + the fleet sleeps. The us-east-1 on-demand G and VT quota (L-DB2E81BA, 384 vCPU) fits 4 g4dn.metal. +- The coordinator's task role may launch EC2 only from the tile and host templates (hosts through `ec2:CreateFleet`), + referencing only their AMIs and the fleet's subnets and group (by exact ARN), only as each template's own types and + tagged `Purpose=fcvm-fleet-tile` or `fcvm-fleet-host`; terminate only instances so tagged; pass only the host and tile + roles; read and write only `state.json`; read the DNS token and the certificate + and store the certificate it issues. The fleet account has EC2's spot and EC2 Fleet service-linked roles. Hosts read the certificate, the agent token and the asset bucket. - Not yet covered: security monitoring like the Dolphin account's. diff --git a/fcvm-fleet.tf b/fcvm-fleet.tf index 60f60cc3..be5a8370 100644 --- a/fcvm-fleet.tf +++ b/fcvm-fleet.tf @@ -4,23 +4,24 @@ # fcvm-fleet member account (fcvm-fleet-account.tf) through its three provider aliases, applied from a jumpbox. # # - modules/fcvm-fleet-region, once per region (us-east-2, us-west-2, us-east-1): the network (TCP 443 in, and the GPU -# hosts' slot ports over IPv4), the spot GPU host group and the EC2 tiles' launch templates. +# hosts' slot ports over IPv4), the GPU host and EC2 tile launch templates. # - The coordinator, in us-east-2: a dedicated ECS cluster and one Fargate service (the coordinator and a cloudflared -# container), its S3 state object, and the task role that may launch and terminate EC2 tiles from the tile templates only. +# container), its S3 state object, and the task role that may launch and terminate GPU hosts and EC2 tiles from their +# templates only. # - Instance roles for the hosts and the tiles, the asset bucket for fcvm release artifacts, the flow-log bucket, and the # wildcard TLS certificate's secret container. The Cloudflare-fed secrets (the DNS token, the coordinator's tunnel token, # the agents' Access service token) are fcvm-dev.tf's, written there with their values; here they are named by ARN. # # TWO GATES, both ON since ejc3/fcvm's fleet AMIs exist (built by fcvm's fleet/ami/build-ami.sh from fcvm-arm): -# - local.fcvm_fleet_hosts_enabled: one spot GPU metal host per region that has a fleet AMI in var.fcvm_fleet_host_amis. Only -# us-east-1 has one (and the only G spot quota), so one g4dn.metal: $4.76-5.44 an hour spot there on 2026-10-22, about -# $3,600 a month always on. A region and architecture without an AMI is left out of the group. -# ON-DEMAND FALLBACK: each region with an AMI also has fcvm-fleet-hosts-on-demand, 0 unless a wake finds no spot capacity -# (modules/fcvm-fleet-region, "On-demand fallback"), capped at local.fcvm_fleet_on_demand_max = 8 hosts (owner, -# 2026-10-24: "Up to 8 on-demand max"). A g4dn.metal on demand is $7.824 an hour in us-east-1, so 8 is $62.59 an hour. -# The account's on-demand G and VT quota there (L-DB2E81BA) is 384 vCPU, 4 hosts at 96 each, less any on-demand EC2 -# tiles (4 vCPU each); the group's cap is 8, and launches beyond the quota fail. The coordinator only falls back with -# local.fcvm_fleet_coordinator_falls_back, which needs a coordinator image that has --on-demand-group. +# - local.fcvm_fleet_hosts_enabled: one spot GPU metal host per region that has a fleet AMI in var.fcvm_fleet_host_amis, which +# the coordinator launches while the fleet is awake and terminates while it sleeps. Only us-east-1 has one (and the only G +# spot quota), so one g4dn.metal: $4.76-5.44 an hour spot there on 2026-10-22. A region and architecture without an AMI +# gets no template. +# ON-DEMAND FALLBACK: when spot is refused for capacity the coordinator launches the host on demand in the same pass +# (modules/fcvm-fleet-region, "GPU hosts"), at most local.fcvm_fleet_on_demand_max = 8 on-demand hosts across the fleet +# (owner, 2026-10-24: "Up to 8 on-demand max"; FLEET_HOST_ON_DEMAND_MAX). A g4dn.metal on demand is $7.824 an hour in +# us-east-1, so 8 is $62.59 an hour. The account's on-demand G and VT quota there (L-DB2E81BA) is 384 vCPU, 4 hosts at 96 +# each, less any on-demand EC2 tiles (4 vCPU each); launches beyond the quota fail. # - local.fcvm_fleet_coordinator_enabled: the coordinator service (one Fargate task, 0.5 vCPU and 1 GB ARM64, about $15 a # month). Its initial image is fcvm-fleet/coordinator:; # main-only CI releases immutable digests through fcvm-fleet-ci.tf. @@ -51,12 +52,14 @@ locals { # The bootstrap image, before main-only CI's first release. Active releases are # digest-pinned in /fcvm-fleet/stage/coordinator-image (fcvm-fleet-ci.tf). fcvm_fleet_coordinator_tag = "bdaa0003" - # Most on-demand GPU hosts per region (the fallback group's max_size; owner, 2026-10-24: "Up to 8 on-demand max"). + # Most on-demand GPU hosts across the fleet (the coordinator's cap; owner, 2026-10-24: "Up to 8 on-demand max"). fcvm_fleet_on_demand_max = 8 - # Whether the coordinator gives a short spot group's capacity to the region's on-demand group (--on-demand-group, - # --spot-wait, ejc3/fcvm#1215). On with the image at c93a01d9 (ejc3/fcvm main, which has those flags); the image at - # 8c96e28e refuses an unknown argument and would not start, so the two change together. - fcvm_fleet_coordinator_falls_back = true + # The coordinator's host launches (fcvm fleet coord, FLEET_HOST_TEMPLATES and FLEET_HOST_SUBNETS, ejc3/fcvm#1257): every + # region's host templates while the gate is on, and the subnets of the AZs that offer a host type. Environment, not + # arguments: an image from before ejc3/fcvm#1257 ignores them, where it would refuse an unknown argument and not start. + fcvm_fleet_host_templates = flatten([for r, m in local.fcvm_fleet_regional : [for t in m.host_template_names : "${r}:${t}"]]) + fcvm_fleet_host_subnets = flatten([for r, m in local.fcvm_fleet_regional : [for s in m.host_subnet_ids : "${r}:${s}"] + if length(m.host_template_names) > 0]) fcvm_fleet_cidrs = { "us-east-2" = "10.20.0.0/16", "us-west-2" = "10.21.0.0/16", "us-east-1" = "10.22.0.0/16" } @@ -116,16 +119,14 @@ locals { "--aggregate-tls-port ${local.fcvm_fleet_aggregate_tls_port}", ] : [])) - # The coordinator's arguments (fcvm fleet/PROTOCOL.md, "Coordinator", "Running it"); its state bucket and key come from - # the task's environment. It owns the host groups' desired capacity: 0 after 10 minutes with no viewer, 1 for the next - # one, moved to the on-demand group when the spot group has no instance 120 s into a wake. The EC2 side runs its tiles in us-east-1, the one region with a tile AMI, and only while the fleet is awake. + # The coordinator's arguments (fcvm fleet/PROTOCOL.md, "Coordinator", "Running it"); its state bucket and key and its + # host templates come from the task's environment. The fleet sleeps after 10 minutes with no viewer (every GPU host + # terminated) and wakes for the next one (one host per region with a template, spot first, on demand at once when spot + # is refused). The EC2 side runs its tiles in us-east-1, the one region with a tile AMI, and only while the fleet is awake. fcvm_fleet_coordinator_command = concat( ["--listen", "0.0.0.0:${local.fcvm_fleet_coordinator_port}", "--trust-cf-connecting-ip", "--allowed-origin", "https://${local.fcvm_dev_site_stage_host}"], - flatten([for r in local.fcvm_fleet_regions : ["--scale-group", "${r}:fcvm-fleet-hosts"] if length(lookup(var.fcvm_fleet_host_amis, r, {})) > 0]), - local.fcvm_fleet_coordinator_falls_back ? concat( - flatten([for r in local.fcvm_fleet_regions : ["--on-demand-group", "${r}:fcvm-fleet-hosts-on-demand"] if length(lookup(var.fcvm_fleet_host_amis, r, {})) > 0]), - ["--spot-wait", "120"], - ) : [], + # Sleeping is implied by the host templates; the flag keeps an image from before ejc3/fcvm#1257 sleeping too. + ["--sleep-when-idle"], ["--ec2", "aws", "--ec2-tiles", "8", "--ec2-regions", "us-east-1", "--ec2-template", "fcvm-fleet-tile-x86_64", "--ec2-on-demand-template", "fcvm-fleet-tile-on-demand-x86_64"], ["--dns-zone", "fcvm.dev", "--dns-token-secret", aws_secretsmanager_secret.fcvm_fleet_dns_token.name, "--dns-zone-id", var.fcvm_dev_zone_id, "--tls-secret", aws_secretsmanager_secret.fcvm_fleet_tls.name, "--secrets-region", local.fcvm_fleet_home_region, "--dns-nameservers", join(",", local.fcvm_dev_nameservers)], # The EC2 half's aggregator beside the coordinator: its layout on loopback, browsers on 443 of the task's own addresses. @@ -376,8 +377,6 @@ module "fcvm_fleet_east2" { hosts_enabled = local.fcvm_fleet_hosts_enabled host_agent_args = local.fcvm_fleet_host_agent_args slot_tls_ports = local.fcvm_fleet_slot_tls_ports - host_on_demand_max = local.fcvm_fleet_on_demand_max - alert_email = local.browser_manager_owner host_profile_name = aws_iam_instance_profile.fcvm_fleet_host.name tile_profile_name = aws_iam_instance_profile.fcvm_fleet_tile.name flow_log_destination = "${aws_s3_bucket.fcvm_fleet_flow_logs.arn}/vpc-flow" @@ -393,8 +392,6 @@ module "fcvm_fleet_west2" { hosts_enabled = local.fcvm_fleet_hosts_enabled host_agent_args = local.fcvm_fleet_host_agent_args slot_tls_ports = local.fcvm_fleet_slot_tls_ports - host_on_demand_max = local.fcvm_fleet_on_demand_max - alert_email = local.browser_manager_owner host_profile_name = aws_iam_instance_profile.fcvm_fleet_host.name tile_profile_name = aws_iam_instance_profile.fcvm_fleet_tile.name flow_log_destination = "${aws_s3_bucket.fcvm_fleet_flow_logs.arn}/vpc-flow" @@ -410,8 +407,6 @@ module "fcvm_fleet_east1" { hosts_enabled = local.fcvm_fleet_hosts_enabled host_agent_args = local.fcvm_fleet_host_agent_args slot_tls_ports = local.fcvm_fleet_slot_tls_ports - host_on_demand_max = local.fcvm_fleet_on_demand_max - alert_email = local.browser_manager_owner host_profile_name = aws_iam_instance_profile.fcvm_fleet_host.name tile_profile_name = aws_iam_instance_profile.fcvm_fleet_tile.name flow_log_destination = "${aws_s3_bucket.fcvm_fleet_flow_logs.arn}/vpc-flow" @@ -442,15 +437,36 @@ locals { "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:${kind}/*" ]]) fcvm_fleet_tile_template_condition = length(local.fcvm_fleet_tile_template_arns) > 0 ? local.fcvm_fleet_tile_template_arns : ["arn:aws:ec2:*:*:launch-template/none"] + fcvm_fleet_tile_types = ["g4dn.xlarge", "g5g.xlarge"] + + # The same for GPU hosts: the host AMIs, the fleet's subnets and group, and the host templates. A host launch creates + # what a tile launch does (instance, volume, network interface, spot request). + fcvm_fleet_host_template_arns = concat(module.fcvm_fleet_east2.host_template_arns, module.fcvm_fleet_west2.host_template_arns, module.fcvm_fleet_east1.host_template_arns) + fcvm_fleet_host_referenced_arns = concat( + flatten([for r, amis in var.fcvm_fleet_host_amis : [for ami in values(amis) : "arn:aws:ec2:${r}::image/${ami}" if ami != ""]]), + flatten([for r, m in local.fcvm_fleet_regional : [for s in m.subnet_ids : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:subnet/${s}"]]), + [for r, m in local.fcvm_fleet_regional : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:security-group/${m.security_group_id}"], + local.fcvm_fleet_host_template_arns, + ["arn:aws:ec2:us-east-1::image/ami-none"], + ) + fcvm_fleet_host_template_condition = length(local.fcvm_fleet_host_template_arns) > 0 ? local.fcvm_fleet_host_template_arns : ["arn:aws:ec2:*:*:launch-template/none"] + fcvm_fleet_host_types = ["g4dn.metal", "g5g.metal"] } -# Spot launches (the tile templates are one-time spot) need EC2's spot service-linked role, which a fresh account lacks -# and the coordinator may not create. +# Spot launches (the tile templates are one-time spot, the hosts spot first) need EC2's spot service-linked role, which a +# fresh account lacks and the coordinator may not create. resource "aws_iam_service_linked_role" "fcvm_fleet_spot" { provider = aws.fcvm_fleet aws_service_name = "spot.amazonaws.com" } +# The GPU hosts are launched as instant EC2 Fleets (CreateFleet), which need EC2 Fleet's service-linked role; the account +# has none (2026-10-25), and the coordinator may not create it. +resource "aws_iam_service_linked_role" "fcvm_fleet_ec2fleet" { + provider = aws.fcvm_fleet + aws_service_name = "ec2fleet.amazonaws.com" +} + # ------------------------------------------------------------------------------------------------ the coordinator's image # The coordinator's container image (fcvm's fleet/coord, ARM64), pushed from fcvm-arm through fcvm-fleet-terraform; the task's # execution role pulls it (AmazonECSTaskExecutionRolePolicy). Tags are fcvm commits and immutable, so a running task's image @@ -544,8 +560,8 @@ resource "aws_iam_role" "fcvm_fleet_coordinator_task" { }) } -# Its state object; the fleet secrets; EC2 tiles from the tile templates only, tagged at launch, and terminated only when -# tagged so; and passing exactly the tile role. In the fleet regions only (the account's SCP denies the others anyway). +# Its state object; the fleet secrets; GPU hosts and EC2 tiles from their templates only, tagged at launch, and terminated +# only when tagged so; and passing exactly the host and tile roles. In the fleet regions only (the account's SCP denies the others anyway). resource "aws_iam_role_policy" "fcvm_fleet_coordinator_task" { provider = aws.fcvm_fleet name = "coordinator" @@ -567,28 +583,6 @@ resource "aws_iam_role_policy" "fcvm_fleet_coordinator_task" { Action = "s3:ListBucket" Resource = aws_s3_bucket.fcvm_fleet_coordinator_state.arn }, - { - # Scale to zero: with no viewer the coordinator sets the host groups' desired capacity to 0 and wakes them for the - # next one (fcvm fleet coord --scale-group :fcvm-fleet-hosts), and falls back to the on-demand group when - # spot has no capacity (--on-demand-group :fcvm-fleet-hosts-on-demand). Only those groups; their max_size - # stays the gate (0 while local.fcvm_fleet_hosts_enabled is false, at most 8 on demand), so the coordinator cannot - # start a host the gate holds off or more on-demand hosts than the cap. - Sid = "ScaleTheHostGroups" - Effect = "Allow" - Action = "autoscaling:SetDesiredCapacity" - Resource = flatten([for r in local.fcvm_fleet_regions : [for g in ["fcvm-fleet-hosts", "fcvm-fleet-hosts-on-demand"] : - "arn:aws:autoscaling:${r}:${local.fcvm_fleet_account_id}:autoScalingGroup:*:autoScalingGroupName/${g}"]]) - }, - { - # Reading the groups' instances while they come up, and their scaling activities: a failed spot launch (Auto - # Scaling records UnfulfillableCapacity or MaxSpotInstanceCountExceeded about 10 s after the raise) moves the - # wake to the on-demand group at once instead of after --spot-wait (ejc3/fcvm#1263). Neither Describe takes a - # resource-level permission. - Sid = "DescribeTheHostGroups" - Effect = "Allow" - Action = ["autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeScalingActivities"] - Resource = "*" - }, { # The DNS token (slot records and the ACME challenge) and the wildcard certificate it issues and renews. Not the # tunnel token (cloudflared gets it through the execution role) nor the agents' Access token. @@ -620,34 +614,81 @@ resource "aws_iam_role_policy" "fcvm_fleet_coordinator_task" { Condition = { ArnLike = { "ec2:LaunchTemplate" = local.fcvm_fleet_tile_template_condition } } }, { - # A launch from a template can still override the instance type: only the two tile types, whatever the request says. - Sid = "TileTypesOnly" + # GPU hosts (fcvm fleet coord --host-template, ejc3/fcvm#1257): an instant EC2 Fleet over the host templates, spot + # first and on demand when spot is refused. EC2 Fleet launches with this role's RunInstances, so the two statements + # below hold it to the host templates and their own resources as the tile statements above do. + Sid = "LaunchHostsAsInstantFleets" + Effect = "Allow" + Action = "ec2:CreateFleet" + Resource = "*" + Condition = { StringEquals = { "aws:RequestedRegion" = local.fcvm_fleet_regions } } + }, + { + Sid = "LaunchHostsFromTheirTemplates" + Effect = "Allow" + Action = "ec2:RunInstances" + Resource = local.fcvm_fleet_tile_created_arns + Condition = { ArnLike = { "ec2:LaunchTemplate" = local.fcvm_fleet_host_template_condition } } + }, + { + # The host AMIs, the fleet's subnets and group, and the host templates. + Sid = "LaunchHostsWithTheirOwnResources" + Effect = "Allow" + Action = "ec2:RunInstances" + Resource = local.fcvm_fleet_host_referenced_arns + Condition = { ArnLike = { "ec2:LaunchTemplate" = local.fcvm_fleet_host_template_condition } } + }, + { + # A launch from a template can still override the instance type: only the fleet's four types, whatever the + # request says, and each template only its own kind (below). + Sid = "FleetTypesOnly" Effect = "Deny" Action = "ec2:RunInstances" Resource = "arn:aws:ec2:*:*:instance/*" - Condition = { StringNotEquals = { "ec2:InstanceType" = ["g4dn.xlarge", "g5g.xlarge"] } } + Condition = { StringNotEquals = { "ec2:InstanceType" = concat(local.fcvm_fleet_tile_types, local.fcvm_fleet_host_types) } } + }, + { + Sid = "TileTemplatesLaunchTileTypesOnly" + Effect = "Deny" + Action = "ec2:RunInstances" + Resource = "arn:aws:ec2:*:*:instance/*" + Condition = { + ArnLike = { "ec2:LaunchTemplate" = local.fcvm_fleet_tile_template_condition } + StringNotEquals = { "ec2:InstanceType" = local.fcvm_fleet_tile_types } + } + }, + { + Sid = "HostTemplatesLaunchHostTypesOnly" + Effect = "Deny" + Action = "ec2:RunInstances" + Resource = "arn:aws:ec2:*:*:instance/*" + Condition = { + ArnLike = { "ec2:LaunchTemplate" = local.fcvm_fleet_host_template_condition } + StringNotEquals = { "ec2:InstanceType" = local.fcvm_fleet_host_types } + } }, { - # The instance a launch creates must carry Purpose=fcvm-fleet-tile, which is what termination is limited to. - Sid = "TileInstancesAreTagged" + # The instance a launch creates must carry Purpose=fcvm-fleet-tile or fcvm-fleet-host, which is what termination + # is limited to. + Sid = "FleetInstancesAreTagged" Effect = "Deny" Action = "ec2:RunInstances" Resource = "arn:aws:ec2:*:*:instance/*" - Condition = { StringNotEquals = { "aws:RequestTag/Purpose" = "fcvm-fleet-tile" } } + Condition = { StringNotEquals = { "aws:RequestTag/Purpose" = ["fcvm-fleet-tile", "fcvm-fleet-host"] } } }, { Sid = "TagAtLaunchOnly" Effect = "Allow" Action = "ec2:CreateTags" Resource = "*" - Condition = { StringEquals = { "ec2:CreateAction" = "RunInstances" } } + Condition = { StringEquals = { "ec2:CreateAction" = ["RunInstances", "CreateFleet"] } } }, { - Sid = "TerminateTilesOnly" + Sid = "TerminateFleetInstancesOnly" Effect = "Allow" Action = "ec2:TerminateInstances" Resource = "arn:aws:ec2:*:*:instance/*" - Condition = { StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } } + Condition = { StringEquals = { "ec2:ResourceTag/Purpose" = ["fcvm-fleet-tile", "fcvm-fleet-host"], "aws:RequestedRegion" = local.fcvm_fleet_regions } } }, { Sid = "DescribeEc2" @@ -656,10 +697,10 @@ resource "aws_iam_role_policy" "fcvm_fleet_coordinator_task" { Resource = "*" }, { - Sid = "PassTheTileRoleOnly" + Sid = "PassTheHostAndTileRolesOnly" Effect = "Allow" Action = "iam:PassRole" - Resource = aws_iam_role.fcvm_fleet_tile.arn + Resource = [aws_iam_role.fcvm_fleet_host.arn, aws_iam_role.fcvm_fleet_tile.arn] Condition = { StringEquals = { "iam:PassedToService" = "ec2.amazonaws.com" } } }, ] @@ -737,6 +778,9 @@ resource "aws_ecs_task_definition" "fcvm_fleet_coordinator" { { name = "FLEET_STATE_BUCKET", value = aws_s3_bucket.fcvm_fleet_coordinator_state.bucket }, { name = "FLEET_STATE_KEY", value = "state.json" }, { name = "FLEET_REGIONS", value = join(",", local.fcvm_fleet_regions) }, + { name = "FLEET_HOST_TEMPLATES", value = join(",", local.fcvm_fleet_host_templates) }, + { name = "FLEET_HOST_SUBNETS", value = join(",", local.fcvm_fleet_host_subnets) }, + { name = "FLEET_HOST_ON_DEMAND_MAX", value = tostring(local.fcvm_fleet_on_demand_max) }, # The coordinator logs its launches, scaling and agent connections at info; without this only warnings reach the log. { name = "RUST_LOG", value = "info" }, ] @@ -811,8 +855,8 @@ output "fcvm_fleet_infra" { state_bucket = aws_s3_bucket.fcvm_fleet_coordinator_state.bucket cluster = aws_ecs_cluster.fcvm_fleet.name coordinator_repo = aws_ecr_repository.fcvm_fleet_coordinator.repository_url - hosts_desired = { "us-east-2" = module.fcvm_fleet_east2.host_group_desired, "us-west-2" = module.fcvm_fleet_west2.host_group_desired, "us-east-1" = module.fcvm_fleet_east1.host_group_desired } - on_demand_max = { "us-east-2" = module.fcvm_fleet_east2.host_on_demand_max, "us-west-2" = module.fcvm_fleet_west2.host_on_demand_max, "us-east-1" = module.fcvm_fleet_east1.host_on_demand_max } + host_templates = local.fcvm_fleet_host_templates + on_demand_max = local.fcvm_fleet_on_demand_max coordinator_on = local.fcvm_fleet_coordinator_running tile_template_arns = local.fcvm_fleet_tile_template_arns } diff --git a/modules/fcvm-fleet-region/main.tf b/modules/fcvm-fleet-region/main.tf index 0f1e82c3..5b04368f 100644 --- a/modules/fcvm-fleet-region/main.tf +++ b/modules/fcvm-fleet-region/main.tf @@ -6,22 +6,20 @@ # - Network: a VPC with an Amazon-provided IPv6 /56, one public subnet per AZ (IPv4 /20 and IPv6 /64), an internet gateway, # and one security group that admits TCP 443 from anywhere and nothing else. No SSH: hosts are reached with SSM Session # Manager. -# - GPU hosts: a launch template per architecture (x86_64 for g4dn.metal, arm64 for g5g.metal), one spot auto scaling -# group over both, and an on-demand group over the same templates that stays at 0 unless the coordinator falls back to it -# (below, "On-demand fallback"). The primary interface asks for a delegated IPv6 /80 (Ipv6PrefixCount = 1) for the host's VMs and no -# single address: EC2 creates an interface with one of an address count or a prefix count, never both (checked -# 2026-10-22). IMDSv2 with hop limit 1, so no VM or container on the host reaches the metadata service. -# - GATE: the group's capacity is 0 unless var.hosts_enabled and at least one architecture has an AMI. The fleet AMI does -# not exist yet, and an always-on metal GPU host is the account's main cost; only types whose architecture has an AMI are -# in the group's overrides. -# - On-demand fallback: Auto Scaling never moves a spot group's capacity to on demand by itself (a mixed instances policy -# keeps its configured split and keeps retrying spot), so the fallback is a second group, fcvm-fleet-hosts-on-demand, -# 100% on demand, max_size var.host_on_demand_max, desired 0. The coordinator owns both groups' desired capacity -# (fcvm fleet coord --on-demand-group): when a wake leaves the spot group short for --spot-wait, it gives the -# on-demand group the shortfall and holds the spot group at what it has; asleep, both are 0. max_size caps what any -# caller can ask for. Its one alarm mails the owner while any on-demand host is in service. -# - Host AZs: both host groups launch only in AZs that offer a host type. us-east-1e has no g4dn.metal, and a spot launch -# there failed with InvalidFleetConfiguration (2026-10-24). +# - GPU hosts: a launch template per architecture (x86_64 for g4dn.metal, arm64 for g5g.metal). No group: the coordinator +# launches and terminates the hosts itself from these templates (fcvm fleet coord --host-template, --host-subnet, +# ejc3/fcvm#1257): spot first with EC2 Fleet (CreateFleet, type instant, over every template and host subnet, price-capacity- +# optimized), and on demand in the same pass when spot is refused for capacity, at most local.fcvm_fleet_on_demand_max +# on-demand hosts across the fleet; every host is terminated while the fleet sleeps. This replaced a spot auto scaling +# group and an on-demand one whose desired capacity the coordinator set: Auto Scaling took 6 to 14 s to start a launch +# after a raise and about 10 s more to record a refused spot launch. The primary interface asks for a delegated IPv6 /80 +# (Ipv6PrefixCount = 1) for the host's VMs and no single address: EC2 creates an interface with one of an address count +# or a prefix count, never both (checked 2026-10-22). IMDSv2 with hop limit 1, so no VM or container on the host reaches +# the metadata service. +# - GATE: the coordinator is given this region's host templates (output host_template_names) only while var.hosts_enabled +# and at least one architecture has an AMI; only types whose architecture has an AMI get a template. +# - Host AZs: hosts launch only in the subnets of AZs that offer a host type (output host_subnet_ids). us-east-1e has no +# g4dn.metal, and a spot launch there failed with InvalidFleetConfiguration (2026-10-24). # - EC2 tiles: a launch template per architecture (g4dn.xlarge, g5g.xlarge) with one IPv6 address; no group, since the # coordinator launches and terminates tiles itself (its task role is limited to these templates). @@ -54,20 +52,6 @@ variable "hosts_enabled" { type = bool } -variable "host_on_demand_max" { - description = "Most on-demand GPU hosts the fallback group may run here (its max_size while hosts are enabled)" - type = number - validation { - condition = var.host_on_demand_max >= 0 && var.host_on_demand_max <= 8 - error_message = "At most 8 on-demand GPU hosts (owner, 2026-10-24)." - } -} - -variable "alert_email" { - description = "Address the on-demand host alarm mails (its subscription waits for a click on AWS's confirmation link)" - type = string -} - variable "host_agent_args" { description = "The host agent's arguments (fcvm fleet agent, FLEET_AGENT_ARGS in /etc/fcvm-fleet/agent.env); empty leaves the agent off" type = string @@ -129,9 +113,8 @@ locals { # The AZs that offer a host type of an architecture with an AMI, and their subnets: both host groups launch there only. host_azs = sort(distinct(flatten([for o in data.aws_ec2_instance_type_offerings.host : o.locations]))) host_subnets = [for az in local.host_azs : aws_subnet.public[az].id if contains(local.azs, az)] - # One host while enabled and an AMI exists; 0 otherwise. max 2 lets a capacity-rebalance replacement start before the old - # host is gone. - host_desired = var.hosts_enabled && length(local.host_archs) > 0 ? 1 : 0 + # Whether the coordinator launches hosts here: enabled and an AMI exists. + hosts_on = var.hosts_enabled && length(local.host_archs) > 0 # The subnet a tile launches in. The coordinator launches from the template without naming a subnet, and an interface # without one lands in the default VPC, where the fleet's security group does not exist and the task role may not # launch (an EC2 tile's RunInstances was refused on the default subnet, 2026-10-24). The first AZ, in name order. @@ -278,150 +261,6 @@ resource "aws_launch_template" "host" { tags = { Name = "fcvm-fleet-host-${each.key}" } } -resource "aws_autoscaling_group" "hosts" { - # Created only once an AMI exists: a mixed instances policy needs at least one launch template. - count = length(local.host_archs) > 0 ? 1 : 0 - - name = "fcvm-fleet-hosts" - vpc_zone_identifier = local.host_subnets - min_size = 0 - max_size = local.host_desired == 0 ? 0 : 2 - desired_capacity = local.host_desired - capacity_rebalance = true - health_check_type = "EC2" - wait_for_capacity_timeout = "0" - - mixed_instances_policy { - instances_distribution { - on_demand_base_capacity = 0 - on_demand_percentage_above_base_capacity = 0 - spot_allocation_strategy = "price-capacity-optimized" - } - - launch_template { - launch_template_specification { - launch_template_id = aws_launch_template.host[local.host_archs[0]].id - version = "$Latest" - } - - dynamic "override" { - for_each = local.host_archs - content { - instance_type = local.host_types[override.value] - launch_template_specification { - launch_template_id = aws_launch_template.host[override.value].id - version = "$Latest" - } - } - } - } - } - - tag { - key = "Purpose" - value = "fcvm-fleet-host" - propagate_at_launch = true - } - - # The coordinator owns the desired capacity once the gate is on (0 with no viewer, back up for the next one); an apply - # must not reset it. max_size above is the gate. - lifecycle { - ignore_changes = [desired_capacity] - } -} - -# The spot group's fallback (header, "On-demand fallback"): the same templates on demand, 0 until the coordinator gives it a -# spot group's shortfall. max_size is the cap, and 0 while the gate is off. -resource "aws_autoscaling_group" "hosts_on_demand" { - count = length(local.host_archs) > 0 ? 1 : 0 - - name = "fcvm-fleet-hosts-on-demand" - vpc_zone_identifier = local.host_subnets - min_size = 0 - max_size = local.host_desired == 0 ? 0 : var.host_on_demand_max - desired_capacity = 0 - health_check_type = "EC2" - wait_for_capacity_timeout = "0" - # Group metrics (free at 1-minute granularity) for the alarm below. - metrics_granularity = "1Minute" - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupPendingInstances"] - - mixed_instances_policy { - instances_distribution { - on_demand_base_capacity = 0 - on_demand_percentage_above_base_capacity = 100 - on_demand_allocation_strategy = "prioritized" - } - - launch_template { - launch_template_specification { - launch_template_id = aws_launch_template.host[local.host_archs[0]].id - version = "$Latest" - } - - dynamic "override" { - for_each = local.host_archs - content { - instance_type = local.host_types[override.value] - launch_template_specification { - launch_template_id = aws_launch_template.host[override.value].id - version = "$Latest" - } - } - } - } - } - - tag { - key = "Purpose" - value = "fcvm-fleet-host" - propagate_at_launch = true - } - - tag { - key = "Market" - value = "on-demand" - propagate_at_launch = true - } - - # The coordinator owns the desired capacity, as the spot group's. - lifecycle { - ignore_changes = [desired_capacity] - } -} - -# An on-demand host is $7.824 an hour in us-east-1 (g4dn.metal), and nothing else reports one: the coordinator keeps it until -# the fleet sleeps, so a viewer that never leaves, or a coordinator that dies while it is up, keeps it running. Mail on the -# first in-service minute of any fallback, and again (OK) when the group is back at 0. -resource "aws_sns_topic" "alerts" { - count = length(local.host_archs) > 0 ? 1 : 0 - name = "fcvm-fleet-alerts" -} - -resource "aws_sns_topic_subscription" "alerts_email" { - count = length(local.host_archs) > 0 ? 1 : 0 - topic_arn = aws_sns_topic.alerts[0].arn - protocol = "email" - endpoint = var.alert_email -} - -resource "aws_cloudwatch_metric_alarm" "on_demand_hosts" { - count = length(local.host_archs) > 0 ? 1 : 0 - alarm_name = "fcvm-fleet-on-demand-hosts" - alarm_description = "fcvm fleet on-demand GPU hosts in service in ${local.region}: spot had no capacity and the coordinator fell back ($7.824/h per g4dn.metal; modules/fcvm-fleet-region)." - comparison_operator = "GreaterThanOrEqualToThreshold" - evaluation_periods = 1 - namespace = "AWS/AutoScaling" - metric_name = "GroupInServiceInstances" - dimensions = { AutoScalingGroupName = aws_autoscaling_group.hosts_on_demand[0].name } - statistic = "Maximum" - period = 300 - threshold = 1 - treat_missing_data = "notBreaching" - alarm_actions = [aws_sns_topic.alerts[0].arn] - ok_actions = [aws_sns_topic.alerts[0].arn] -} - # ------------------------------------------------------------------------------------------------ EC2 tiles resource "aws_launch_template" "tile" { for_each = toset(local.tile_archs) @@ -519,10 +358,21 @@ output "tile_template_arns" { value = concat([for t in aws_launch_template.tile : t.arn], [for t in aws_launch_template.tile_on_demand : t.arn]) } -output "host_group_desired" { - value = local.host_desired +# The coordinator's host launches (fcvm fleet coord --host-template :, --host-subnet :): the +# templates only while the gate is on, the subnets of the AZs that offer a host type. +output "host_template_names" { + value = local.hosts_on ? [for a in local.host_archs : aws_launch_template.host[a].name] : [] +} + +output "host_subnet_ids" { + value = local.host_subnets +} + +# The coordinator's RunInstances (through CreateFleet) is limited to these templates and the host AMIs. +output "host_template_arns" { + value = [for t in aws_launch_template.host : t.arn] } -output "host_on_demand_max" { - value = length(local.host_archs) > 0 ? aws_autoscaling_group.hosts_on_demand[0].max_size : 0 +output "host_types" { + value = [for a in local.host_archs : local.host_types[a]] } diff --git a/scripts/test-fcvm-fleet.py b/scripts/test-fcvm-fleet.py index 06b123f9..de0dc36d 100644 --- a/scripts/test-fcvm-fleet.py +++ b/scripts/test-fcvm-fleet.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Offline tests for fcvm-fleet.tf and modules/fcvm-fleet-region (ejc3/aws#606): the fleet's network admits TCP 443 only, -the GPU hosts and the coordinator stay off until their gates and inputs exist, and the coordinator's role reaches EC2 tiles -only.""" +the GPU hosts and the coordinator stay off until their gates and inputs exist, and the coordinator's role reaches GPU hosts +and EC2 tiles from their templates only.""" import re import unittest from pathlib import Path @@ -55,14 +55,14 @@ def test_hosts_ask_for_a_prefix_only_and_imdsv2_hop_1(self): self.assertIn('http_tokens = "required"', lt) self.assertIn("http_put_response_hop_limit = 1", lt) - def test_the_host_group_is_spot_price_capacity_optimized_and_gated(self): - asg = block('resource "aws_autoscaling_group" "hosts"', MOD) - self.assertIn("on_demand_base_capacity = 0", asg) - self.assertIn("on_demand_percentage_above_base_capacity = 0", asg) - self.assertIn('spot_allocation_strategy = "price-capacity-optimized"', asg) - self.assertIn("capacity_rebalance = true", asg) - self.assertIn("desired_capacity = local.host_desired", asg) - self.assertIn("host_desired = var.hosts_enabled && length(local.host_archs) > 0 ? 1 : 0", MOD) + def test_the_coordinator_launches_the_hosts_and_no_group_does(self): + # Auto Scaling took 6 to 14 s to start a launch after a raise (ejc3/fcvm#1257); the coordinator launches the hosts + # itself from the templates, which it gets only while the gate is on. + self.assertNotIn("aws_autoscaling_group", MOD) + self.assertNotIn("aws_autoscaling_group", TF) + self.assertIn("hosts_on = var.hosts_enabled && length(local.host_archs) > 0", MOD) + names = block('output "host_template_names"', MOD) + self.assertIn("value = local.hosts_on ? [for a in local.host_archs : aws_launch_template.host[a].name] : []", names) self.assertIn("fcvm_fleet_hosts_enabled = true", TF) def test_hosts_and_tiles_have_amis_only_where_there_is_g_spot_quota(self): @@ -72,67 +72,41 @@ def test_hosts_and_tiles_have_amis_only_where_there_is_g_spot_quota(self): default = re.search(r"default = (\{.*\})\n", block('variable "%s"' % var)).group(1) self.assertRegex(default, r'^\{ "us-east-1" = \{ x86_64 = "ami-[0-9a-f]{17}" \} \}$') - def test_only_architectures_with_an_ami_are_in_the_group(self): + def test_only_architectures_with_an_ami_get_a_template(self): self.assertIn('host_archs = [for a in sort(keys(local.host_types)) : a if try(var.gpu_amis[a], "") != ""]', MOD) self.assertIn('host_types = { x86_64 = "g4dn.metal", arm64 = "g5g.metal" }', MOD) class OnDemandFallbackTests(unittest.TestCase): - """The spot group's on-demand fallback: a second group, 0 until the coordinator gives it a short spot group's capacity, - capped at 8 hosts (owner, 2026-10-24: "Up to 8 on-demand max").""" - - def test_the_fallback_group_is_on_demand_capped_and_starts_at_zero(self): - od = block('resource "aws_autoscaling_group" "hosts_on_demand"', MOD) - self.assertIn('name = "fcvm-fleet-hosts-on-demand"', od) - self.assertIn("on_demand_percentage_above_base_capacity = 100", od) - self.assertIn("min_size = 0", od) - self.assertIn("desired_capacity = 0", od) - self.assertIn("max_size = local.host_desired == 0 ? 0 : var.host_on_demand_max", od) - self.assertIn("ignore_changes = [desired_capacity]", od) - self.assertNotIn("capacity_rebalance", od) # a spot-only setting - self.assertIn("var.host_on_demand_max >= 0 && var.host_on_demand_max <= 8", block('variable "host_on_demand_max"', MOD)) - self.assertIn("fcvm_fleet_on_demand_max = 8", TF) - self.assertEqual(TF.count("host_on_demand_max = local.fcvm_fleet_on_demand_max"), 3) + """Spot first, on demand in the same pass when spot is refused, at most 8 on-demand hosts across the fleet (owner, + 2026-10-24: "Up to 8 on-demand max"): the coordinator's cap, handed to it in its environment.""" - def test_eight_is_the_cap_for_the_whole_fleet(self): - # The cap is per region, so it is the fleet's cap only while one region has a host AMI (and so a group). - default = re.search(r"default = (\{.*\})\n", block('variable "fcvm_fleet_host_amis"')).group(1) - self.assertEqual(default.count("ami-"), 1) - self.assertIn("count = length(local.host_archs) > 0 ? 1 : 0", block('resource "aws_autoscaling_group" "hosts_on_demand"', MOD)) + def test_the_on_demand_cap_is_eight_for_the_whole_fleet(self): + self.assertIn("fcvm_fleet_on_demand_max = 8", TF) + td = block('resource "aws_ecs_task_definition" "fcvm_fleet_coordinator"') + self.assertIn('{ name = "FLEET_HOST_ON_DEMAND_MAX", value = tostring(local.fcvm_fleet_on_demand_max) }', td) + self.assertNotIn("host_on_demand_max", MOD) - def test_both_host_groups_launch_only_where_a_host_type_is_offered(self): - for name in ('resource "aws_autoscaling_group" "hosts"', 'resource "aws_autoscaling_group" "hosts_on_demand"'): - self.assertIn("vpc_zone_identifier = local.host_subnets", block(name, MOD), name) + def test_hosts_launch_only_where_a_host_type_is_offered(self): + self.assertIn("value = local.host_subnets", block('output "host_subnet_ids"', MOD)) offerings = block('data "aws_ec2_instance_type_offerings" "host"', MOD) self.assertIn('location_type = "availability-zone"', offerings) self.assertIn("values = [local.host_types[each.key]]", offerings) + subnets = TF.split("fcvm_fleet_host_subnets = flatten(", 1)[1].split("\n\n", 1)[0] + self.assertIn('[for s in m.host_subnet_ids : "${r}:${s}"]', subnets) + self.assertIn("if length(m.host_template_names) > 0", subnets) - def test_the_coordinator_falls_back_only_with_an_image_that_has_the_flags(self): - # The image at 8c96e28e has no --on-demand-group; clap refuses an unknown argument and the coordinator would not - # start. The flag may only be turned on together with a newer tag. - tag = re.search(r'fcvm_fleet_coordinator_tag = "([0-9a-f]+)"', TF).group(1) - on = re.search(r"fcvm_fleet_coordinator_falls_back = (true|false)", TF).group(1) - if tag == "8c96e28e": - self.assertEqual(on, "false") + def test_the_host_templates_reach_the_coordinator_in_its_environment(self): + # An image from before ejc3/fcvm#1257 refuses an unknown argument and does not start, and the image after it has no + # --scale-group: the templates go in the environment, which the older image ignores, so either image starts. + td = block('resource "aws_ecs_task_definition" "fcvm_fleet_coordinator"') + self.assertIn('{ name = "FLEET_HOST_TEMPLATES", value = join(",", local.fcvm_fleet_host_templates) }', td) + self.assertIn('{ name = "FLEET_HOST_SUBNETS", value = join(",", local.fcvm_fleet_host_subnets) }', td) + self.assertIn('[for t in m.host_template_names : "${r}:${t}"]', TF) cmd = TF.split("fcvm_fleet_coordinator_command = concat(", 1)[1].split("\n )\n", 1)[0] - self.assertIn("local.fcvm_fleet_coordinator_falls_back ? concat(", cmd) - self.assertIn('["--on-demand-group", "${r}:fcvm-fleet-hosts-on-demand"] if length(lookup(var.fcvm_fleet_host_amis, r, {})) > 0', cmd) - - def test_the_coordinator_may_scale_the_on_demand_group(self): - tf = TF.split('Sid = "ScaleTheHostGroups"', 1)[1].split("},", 1)[0] - self.assertIn('for g in ["fcvm-fleet-hosts", "fcvm-fleet-hosts-on-demand"]', tf) - self.assertIn('Action = "autoscaling:SetDesiredCapacity"', tf) - - def test_an_on_demand_host_in_service_mails_the_owner(self): - alarm = block('resource "aws_cloudwatch_metric_alarm" "on_demand_hosts"', MOD) - self.assertIn('metric_name = "GroupInServiceInstances"', alarm) - self.assertIn("AutoScalingGroupName = aws_autoscaling_group.hosts_on_demand[0].name", alarm) - self.assertIn("threshold = 1", alarm) - self.assertIn("alarm_actions = [aws_sns_topic.alerts[0].arn]", alarm) - self.assertIn('"GroupInServiceInstances"', block('resource "aws_autoscaling_group" "hosts_on_demand"', MOD)) - sub = block('resource "aws_sns_topic_subscription" "alerts_email"', MOD) - self.assertIn('protocol = "email"', sub) - self.assertEqual(TF.count("alert_email = local.browser_manager_owner"), 3) + self.assertIn('["--sleep-when-idle"]', cmd) + for gone in ("--scale-group", "--on-demand-group", "--spot-wait", "--host-template"): + self.assertNotIn(gone, cmd) class CoordinatorTests(unittest.TestCase): @@ -145,13 +119,13 @@ def test_the_service_runs_the_tagged_image_from_the_fleet_repository(self): self.assertIn("image = local.fcvm_fleet_coordinator_running ? nonsensitive(data.aws_ssm_parameter.fcvm_coordinator_release.value)", TF) self.assertIn("desired_count = local.fcvm_fleet_coordinator_running ? 1 : 0", block('resource "aws_ecs_service" "fcvm_fleet_coordinator"')) - def test_the_task_role_reaches_tiles_only(self): + def test_the_task_role_reaches_hosts_and_tiles_only(self): pol = block('resource "aws_iam_role_policy" "fcvm_fleet_coordinator_task"') self.assertIn('"ec2:LaunchTemplate"', pol) - self.assertIn('Condition = { StringNotEquals = { "aws:RequestTag/Purpose" = "fcvm-fleet-tile" } }', pol) - self.assertIn('Condition = { StringEquals = { "ec2:CreateAction" = "RunInstances" } }', pol) - self.assertIn('"ec2:ResourceTag/Purpose" = "fcvm-fleet-tile"', pol) - self.assertIn("Resource = aws_iam_role.fcvm_fleet_tile.arn", pol) + self.assertIn('Condition = { StringNotEquals = { "aws:RequestTag/Purpose" = ["fcvm-fleet-tile", "fcvm-fleet-host"] } }', pol) + self.assertIn('Condition = { StringEquals = { "ec2:CreateAction" = ["RunInstances", "CreateFleet"] } }', pol) + self.assertIn('"ec2:ResourceTag/Purpose" = ["fcvm-fleet-tile", "fcvm-fleet-host"]', pol) + self.assertIn("Resource = [aws_iam_role.fcvm_fleet_host.arn, aws_iam_role.fcvm_fleet_tile.arn]", pol) self.assertIn('"iam:PassedToService" = "ec2.amazonaws.com"', pol) self.assertNotRegex(pol, r'Action\s*=\s*"(ec2|iam|s3):\*"') @@ -217,18 +191,26 @@ def test_a_missing_state_object_reads_as_404(self): self.assertIn('Action = "s3:ListBucket"', s) self.assertIn("Resource = aws_s3_bucket.fcvm_fleet_coordinator_state.arn\n", s + "\n") - def test_the_coordinator_scales_only_the_host_groups(self): - s = self.stmt("ScaleTheHostGroups") - self.assertIn('Action = "autoscaling:SetDesiredCapacity"', s) - self.assertIn(':autoScalingGroup:*:autoScalingGroupName/${g}"', s) - self.assertIn('[for g in ["fcvm-fleet-hosts", "fcvm-fleet-hosts-on-demand"] :', s) - self.assertNotIn('Resource = "*"', s) - d = self.stmt("DescribeTheHostGroups") - self.assertIn('Action = ["autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeScalingActivities"]', d) - self.assertNotRegex(self.TF, r'"autoscaling:\*"') - group = block('resource "aws_autoscaling_group" "hosts"', MOD) - self.assertIn("ignore_changes = [desired_capacity]", group) - self.assertIn("max_size = local.host_desired == 0 ? 0 : 2", group) + def test_the_coordinator_launches_hosts_only_from_their_templates(self): + fleet = self.stmt("LaunchHostsAsInstantFleets") + self.assertIn('Action = "ec2:CreateFleet"', fleet) + self.assertIn('"aws:RequestedRegion" = local.fcvm_fleet_regions', fleet) + created = self.stmt("LaunchHostsFromTheirTemplates") + referenced = self.stmt("LaunchHostsWithTheirOwnResources") + self.assertIn("Resource = local.fcvm_fleet_host_referenced_arns", referenced) + for s in (created, referenced): + self.assertIn('"ec2:LaunchTemplate" = local.fcvm_fleet_host_template_condition', s) + refs = self.TF.split("fcvm_fleet_host_referenced_arns = concat(", 1)[1].split("\n )\n", 1)[0] + for part in ("var.fcvm_fleet_host_amis", ":subnet/${s}", ":security-group/${m.security_group_id}", "local.fcvm_fleet_host_template_arns"): + self.assertIn(part, refs, part) + # A host template launches host types only, and EC2 Fleet's service-linked role exists. + deny = self.stmt("HostTemplatesLaunchHostTypesOnly") + self.assertIn('Effect = "Deny"', deny) + self.assertIn('ArnLike = { "ec2:LaunchTemplate" = local.fcvm_fleet_host_template_condition }', deny) + self.assertIn('StringNotEquals = { "ec2:InstanceType" = local.fcvm_fleet_host_types }', deny) + self.assertIn('fcvm_fleet_host_types = ["g4dn.metal", "g5g.metal"]', self.TF) + self.assertIn('aws_service_name = "ec2fleet.amazonaws.com"', self.TF) + self.assertNotIn("autoscaling:", self.TF) def test_an_on_demand_tile_template_is_a_launchable_template(self): od = block('resource "aws_launch_template" "tile_on_demand"', MOD) @@ -254,12 +236,16 @@ def test_no_module_level_depends_on(self): self.assertIn("flow_log_bucket_policy_id = aws_s3_bucket_policy.fcvm_fleet_flow_logs.id", body, name) class CoordinatorTileTypes(unittest.TestCase): - def test_the_coordinator_launches_only_the_two_tile_types(self): - tf = (ROOT / "fcvm-fleet.tf").read_text() - s = tf.split('Sid = "TileTypesOnly"', 1)[1].split("},", 1)[0] + def test_the_coordinator_launches_only_the_fleet_types_and_tile_templates_only_tile_types(self): + stmt = CoordinatorLaunchSplit.stmt + s = stmt(CoordinatorLaunchSplit, "FleetTypesOnly") self.assertIn('Effect = "Deny"', s) self.assertIn('Action = "ec2:RunInstances"', s) - self.assertIn('"ec2:InstanceType" = ["g4dn.xlarge", "g5g.xlarge"]', s) + self.assertIn('"ec2:InstanceType" = concat(local.fcvm_fleet_tile_types, local.fcvm_fleet_host_types)', s) + t = stmt(CoordinatorLaunchSplit, "TileTemplatesLaunchTileTypesOnly") + self.assertIn('ArnLike = { "ec2:LaunchTemplate" = local.fcvm_fleet_tile_template_condition }', t) + self.assertIn('StringNotEquals = { "ec2:InstanceType" = local.fcvm_fleet_tile_types }', t) + self.assertIn('fcvm_fleet_tile_types = ["g4dn.xlarge", "g5g.xlarge"]', CoordinatorLaunchSplit.TF) class RegistryTests(unittest.TestCase): def test_the_bootstrap_image_is_protected_from_the_rolling_retention_rule(self): @@ -304,9 +290,9 @@ def test_boot_writes_the_agent_env_and_starts_the_unit_only_with_arguments(self) self.assertIn("systemctl start --no-block fcvm-fleet-agent.service", MOD) self.assertNotIn("systemctl enable --now fcvm-fleet-agent", MOD) - def test_the_coordinator_scales_the_host_groups_it_has_an_ami_for(self): + def test_the_coordinator_sleeps_and_runs_the_ec2_side(self): cmd = TF.split("fcvm_fleet_coordinator_command = concat(", 1)[1].split("\n )\n", 1)[0] - self.assertIn('["--scale-group", "${r}:fcvm-fleet-hosts"] if length(lookup(var.fcvm_fleet_host_amis, r, {})) > 0', cmd) + self.assertIn('["--sleep-when-idle"]', cmd) self.assertIn('"--allowed-origin", "https://${local.fcvm_dev_site_stage_host}"', cmd) self.assertIn('"--ec2-regions", "us-east-1"', cmd) self.assertIn('"--dns-token-secret", aws_secretsmanager_secret.fcvm_fleet_dns_token.name', cmd) From 721185a5207faff8e125cd646fc84d194ee4c186 Mon Sep 17 00:00:00 2001 From: ejc3 Date: Mon, 26 Oct 2026 02:30:42 +0000 Subject: [PATCH 3/5] fix: authorize tagged tile spot requests with supported context --- fcvm-fleet.tf | 18 ++++++++++++++++-- scripts/test-fcvm-fleet.py | 7 ++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/fcvm-fleet.tf b/fcvm-fleet.tf index 18a5c77b..e57e2c7b 100644 --- a/fcvm-fleet.tf +++ b/fcvm-fleet.tf @@ -442,11 +442,13 @@ locals { local.fcvm_fleet_tile_template_arns, ["arn:aws:ec2:us-east-1::image/ami-none"], ) - # What a tile launch creates: the instance, its volume, its network interface and its one-time spot request. + # Created resources with launch-template context. A tagged spot request has no ec2:LaunchTemplate context and gets + # its own region/Purpose-scoped authorization below; putting that condition on its ARN rejects persistent tile pools. fcvm_fleet_tile_created_arns = flatten([for r in local.fcvm_fleet_regions : [ - for kind in ["instance", "volume", "network-interface", "spot-instances-request"] : + for kind in ["instance", "volume", "network-interface"] : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:${kind}/*" ]]) + fcvm_fleet_tile_spot_request_arns = [for r in local.fcvm_fleet_regions : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:spot-instances-request/*"] fcvm_fleet_tile_template_condition = length(local.fcvm_fleet_tile_template_arns) > 0 ? local.fcvm_fleet_tile_template_arns : ["arn:aws:ec2:*:*:launch-template/none"] } @@ -622,6 +624,18 @@ resource "aws_iam_role_policy" "fcvm_fleet_coordinator_task" { Resource = local.fcvm_fleet_tile_referenced_arns Condition = { ArnLike = { "ec2:LaunchTemplate" = local.fcvm_fleet_tile_template_condition } } }, + { + # RunInstances checks a tagged persistent spot request separately and that resource has no LaunchTemplate key. + # The instance/AMI/subnet/interface still need their template-scoped grants; this cannot authorize a launch alone. + Sid = "LaunchTaggedTileSpotRequestsOnly" + Effect = "Allow" + Action = "ec2:RunInstances" + Resource = local.fcvm_fleet_tile_spot_request_arns + Condition = { StringEquals = { + "aws:RequestedRegion" = local.fcvm_fleet_regions + "aws:RequestTag/Purpose" = "fcvm-fleet-tile" + } } + }, { # A launch from a template can still override the instance type: only the two tile types, whatever the request says. Sid = "TileTypesOnly" diff --git a/scripts/test-fcvm-fleet.py b/scripts/test-fcvm-fleet.py index c20131e7..f3c63885 100644 --- a/scripts/test-fcvm-fleet.py +++ b/scripts/test-fcvm-fleet.py @@ -197,7 +197,12 @@ def test_created_and_referenced_resources_are_split_and_pinned(self): refs = self.TF.split("fcvm_fleet_tile_referenced_arns = concat(", 1)[1].split("\n )\n", 1)[0] for part in (":image/${ami}", ":subnet/${s}", ":security-group/${m.security_group_id}", "local.fcvm_fleet_tile_template_arns"): self.assertIn(part, refs, part) - self.assertIn('"instance", "volume", "network-interface", "spot-instances-request"', self.TF) + self.assertIn('"instance", "volume", "network-interface"', self.TF) + spot = self.stmt("LaunchTaggedTileSpotRequestsOnly") + self.assertIn('Resource = local.fcvm_fleet_tile_spot_request_arns', spot) + self.assertIn('"aws:RequestTag/Purpose" = "fcvm-fleet-tile"', spot) + self.assertIn('"aws:RequestedRegion"', spot) + self.assertNotIn('"ec2:LaunchTemplate"', spot) def test_state_object_secrets_and_spot_role(self): self.assertIn('Resource = "${aws_s3_bucket.fcvm_fleet_coordinator_state.arn}/state.json"', self.TF) From 36328ec764e34c2a965f837056fb279e704bba49 Mon Sep 17 00:00:00 2001 From: ejc3 Date: Mon, 26 Oct 2026 02:40:38 +0000 Subject: [PATCH 4/5] fix: scope fleet launches and preserve on-demand cost alerts --- AGENTS.md | 8 +- fcvm-fleet.tf | 194 +++++++++++++++++++++---- modules/fcvm-fleet-region/main.tf | 30 +++- scripts/fcvm-fleet-on-demand-notify.py | 22 +++ scripts/test-fcvm-fleet.py | 107 +++++++++++++- 5 files changed, 324 insertions(+), 37 deletions(-) create mode 100644 scripts/fcvm-fleet-on-demand-notify.py diff --git a/AGENTS.md b/AGENTS.md index c8e25a44..431b9653 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1727,12 +1727,16 @@ trusts anything in `fcvm-fleet`, and no peering, shared subnet or resource polic EC2 Fleet over the region's host templates and host subnets, `FLEET_HOST_TEMPLATES`, `FLEET_HOST_SUBNETS`), on demand in the same pass when spot is refused for capacity, at most `local.fcvm_fleet_on_demand_max` = 8 on-demand hosts across the fleet (owner, 2026-10-24: "Up to 8 on-demand max"; `FLEET_HOST_ON_DEMAND_MAX`), and terminates every host while - the fleet sleeps. The us-east-1 on-demand G and VT quota (L-DB2E81BA, 384 vCPU) fits 4 g4dn.metal. + the fleet sleeps. The us-east-1 on-demand G and VT quota (L-DB2E81BA, 384 vCPU) fits 4 g4dn.metal. Accepted on-demand + launches (including recovered idempotent responses) increment `FCVM/Fleet:OnDemandHostLaunches`; the coordinator's + `fcvm-fleet-on-demand-host-launches` alarm notifies the existing `fcvm-fleet-alerts` subscribers. This is a launch-event + notification: its return to OK does not mean running hosts stopped, so it sends no OK notification. - The coordinator's task role may launch EC2 only from the tile and host templates (hosts through `ec2:CreateFleet`), referencing only their AMIs and the fleet's subnets and group (by exact ARN), only as each template's own types and tagged `Purpose=fcvm-fleet-tile` or `fcvm-fleet-host`; terminate only instances so tagged; pass only the host and tile roles; read and write only `state.json`; read the DNS token and the certificate - and store the certificate it issues. The fleet account has EC2's spot and EC2 Fleet service-linked roles. Hosts read the certificate, + and store the certificate it issues. The fleet account has EC2's spot service-linked role; instant EC2 Fleets need no + EC2 Fleet service-linked role. Hosts read the certificate, the agent token and the asset bucket. - Not yet covered: security monitoring like the Dolphin account's. diff --git a/fcvm-fleet.tf b/fcvm-fleet.tf index 6924c151..f882e25b 100644 --- a/fcvm-fleet.tf +++ b/fcvm-fleet.tf @@ -53,7 +53,9 @@ locals { # digest-pinned in /fcvm-fleet/stage/coordinator-image (fcvm-fleet-ci.tf). fcvm_fleet_coordinator_tag = "bdaa0003" # Most on-demand GPU hosts across the fleet (the coordinator's cap; owner, 2026-10-24: "Up to 8 on-demand max"). - fcvm_fleet_on_demand_max = 8 + fcvm_fleet_on_demand_max = 8 + fcvm_fleet_on_demand_alarm_arn = "arn:aws:cloudwatch:${local.fcvm_fleet_home_region}:${local.fcvm_fleet_account_id}:alarm:fcvm-fleet-on-demand-host-launches" + fcvm_fleet_alert_topic_arns = concat(module.fcvm_fleet_east2.alert_topic_arns, module.fcvm_fleet_west2.alert_topic_arns, module.fcvm_fleet_east1.alert_topic_arns) # The coordinator's host launches (fcvm fleet coord, FLEET_HOST_TEMPLATES and FLEET_HOST_SUBNETS, ejc3/fcvm#1257): every # region's host templates while the gate is on, and the subnets of the AZs that offer a host type. Environment, not # arguments: an image from before ejc3/fcvm#1257 ignores them, where it would refuse an unknown argument and not start. @@ -375,6 +377,7 @@ module "fcvm_fleet_east2" { gpu_amis = lookup(var.fcvm_fleet_host_amis, "us-east-2", {}) tile_amis = lookup(var.fcvm_fleet_tile_amis, "us-east-2", {}) hosts_enabled = local.fcvm_fleet_hosts_enabled + alert_email = local.browser_manager_owner host_agent_args = local.fcvm_fleet_host_agent_args slot_tls_ports = local.fcvm_fleet_slot_tls_ports host_profile_name = aws_iam_instance_profile.fcvm_fleet_host.name @@ -390,6 +393,7 @@ module "fcvm_fleet_west2" { gpu_amis = lookup(var.fcvm_fleet_host_amis, "us-west-2", {}) tile_amis = lookup(var.fcvm_fleet_tile_amis, "us-west-2", {}) hosts_enabled = local.fcvm_fleet_hosts_enabled + alert_email = local.browser_manager_owner host_agent_args = local.fcvm_fleet_host_agent_args slot_tls_ports = local.fcvm_fleet_slot_tls_ports host_profile_name = aws_iam_instance_profile.fcvm_fleet_host.name @@ -405,6 +409,7 @@ module "fcvm_fleet_east1" { gpu_amis = lookup(var.fcvm_fleet_host_amis, "us-east-1", {}) tile_amis = lookup(var.fcvm_fleet_tile_amis, "us-east-1", {}) hosts_enabled = local.fcvm_fleet_hosts_enabled + alert_email = local.browser_manager_owner host_agent_args = local.fcvm_fleet_host_agent_args slot_tls_ports = local.fcvm_fleet_slot_tls_ports host_profile_name = aws_iam_instance_profile.fcvm_fleet_host.name @@ -431,26 +436,32 @@ locals { local.fcvm_fleet_tile_template_arns, ["arn:aws:ec2:us-east-1::image/ami-none"], ) - # What a tile launch creates: the instance, its volume, its network interface and its one-time spot request. + # Created resources with launch-template context. A tagged spot request has no ec2:LaunchTemplate context and gets + # its own region/Purpose-scoped authorization below; putting that condition on its ARN rejects persistent tile pools. fcvm_fleet_tile_created_arns = flatten([for r in local.fcvm_fleet_regions : [ - for kind in ["instance", "volume", "network-interface", "spot-instances-request"] : + for kind in ["instance", "volume", "network-interface"] : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:${kind}/*" ]]) + fcvm_fleet_tile_spot_request_arns = [for r in local.fcvm_fleet_regions : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:spot-instances-request/*"] fcvm_fleet_tile_template_condition = length(local.fcvm_fleet_tile_template_arns) > 0 ? local.fcvm_fleet_tile_template_arns : ["arn:aws:ec2:*:*:launch-template/none"] fcvm_fleet_tile_types = ["g4dn.xlarge", "g5g.xlarge"] - # The same for GPU hosts: the host AMIs, the fleet's subnets and group, and the host templates. A host launch creates - # what a tile launch does (instance, volume, network interface, spot request). + # The same for GPU hosts: their AMIs, host-eligible fleet subnets and groups, and enabled host templates. fcvm_fleet_host_template_arns = concat(module.fcvm_fleet_east2.host_template_arns, module.fcvm_fleet_west2.host_template_arns, module.fcvm_fleet_east1.host_template_arns) fcvm_fleet_host_referenced_arns = concat( flatten([for r, amis in var.fcvm_fleet_host_amis : [for ami in values(amis) : "arn:aws:ec2:${r}::image/${ami}" if ami != ""]]), - flatten([for r, m in local.fcvm_fleet_regional : [for s in m.subnet_ids : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:subnet/${s}"]]), - [for r, m in local.fcvm_fleet_regional : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:security-group/${m.security_group_id}"], + flatten([for r, m in local.fcvm_fleet_regional : [for s in m.host_subnet_ids : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:subnet/${s}"] if length(m.host_template_names) > 0]), + [for r, m in local.fcvm_fleet_regional : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:security-group/${m.security_group_id}" if length(m.host_template_names) > 0], local.fcvm_fleet_host_template_arns, ["arn:aws:ec2:us-east-1::image/ami-none"], ) fcvm_fleet_host_template_condition = length(local.fcvm_fleet_host_template_arns) > 0 ? local.fcvm_fleet_host_template_arns : ["arn:aws:ec2:*:*:launch-template/none"] fcvm_fleet_host_types = ["g4dn.metal", "g5g.metal"] + # CreateFleet authorizes the fleet and its newly created instances/volumes separately from the resources in the + # launch template (those still need RunInstances). Keep instance-only conditions off fleet and volume resources. + fcvm_fleet_host_fleet_arns = [for r in local.fcvm_fleet_regions : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:fleet/*"] + fcvm_fleet_host_instance_arns = [for r in local.fcvm_fleet_regions : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:instance/*"] + fcvm_fleet_host_volume_arns = [for r in local.fcvm_fleet_regions : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:volume/*"] } # Spot launches (the tile templates are one-time spot, the hosts spot first) need EC2's spot service-linked role, which a @@ -460,12 +471,8 @@ resource "aws_iam_service_linked_role" "fcvm_fleet_spot" { aws_service_name = "spot.amazonaws.com" } -# The GPU hosts are launched as instant EC2 Fleets (CreateFleet), which need EC2 Fleet's service-linked role; the account -# has none (2026-10-25), and the coordinator may not create it. -resource "aws_iam_service_linked_role" "fcvm_fleet_ec2fleet" { - provider = aws.fcvm_fleet - aws_service_name = "ec2fleet.amazonaws.com" -} +# Instant EC2 Fleets use the caller's permissions and need no EC2 Fleet service-linked role. Do not create the role +# used by persistent request/maintain fleets: the coordinator owns the lifetime of each launched host itself. # ------------------------------------------------------------------------------------------------ the coordinator's image # The coordinator's container image (fcvm's fleet/coord, ARM64), pushed from fcvm-arm through fcvm-fleet-terraform; the task's @@ -521,6 +528,116 @@ resource "aws_cloudwatch_log_group" "fcvm_fleet_coordinator" { retention_in_days = 30 } +# Count only accepted on-demand launches, including an accepted response recovered with its idempotency token +# (ejc3/fcvm#1282, src/fleet/coord/hosts.rs). The earlier "launching on demand now" line is only an attempt. +# This is an event notification, not a count of running hosts; a quiet minute does not mean an expensive host stopped. +resource "aws_cloudwatch_log_metric_filter" "fcvm_fleet_on_demand_launches" { + provider = aws.fcvm_fleet + name = "fcvm-fleet-on-demand-host-launches" + log_group_name = aws_cloudwatch_log_group.fcvm_fleet_coordinator.name + pattern = "\"wake: host launch accepted\" \"on-demand\"" + + metric_transformation { + name = "OnDemandHostLaunches" + namespace = "FCVM/Fleet" + value = "1" + default_value = 0 + unit = "Count" + } +} + +resource "aws_cloudwatch_metric_alarm" "fcvm_fleet_on_demand_launches" { + provider = aws.fcvm_fleet + alarm_name = "fcvm-fleet-on-demand-host-launches" + alarm_description = "The coordinator accepted an on-demand GPU host launch. A g4dn.metal costs $7.824/hour in us-east-1 and runs until the fleet sleeps; inspect the coordinator hosts and logs. This counts launch events, not running hosts." + comparison_operator = "GreaterThanOrEqualToThreshold" + evaluation_periods = 1 + namespace = aws_cloudwatch_log_metric_filter.fcvm_fleet_on_demand_launches.metric_transformation[0].namespace + metric_name = aws_cloudwatch_log_metric_filter.fcvm_fleet_on_demand_launches.metric_transformation[0].name + statistic = "Sum" + period = 60 + threshold = 1 + treat_missing_data = "notBreaching" + alarm_actions = [aws_lambda_function.fcvm_fleet_on_demand_notify.arn] + depends_on = [aws_lambda_permission.fcvm_fleet_on_demand_notify] +} + +# Alarm actions stay in the coordinator's region. This small forwarder publishes through each retained topic's SNS +# regional endpoint, preserving the existing confirmed subscriber without relying on cross-region alarm actions. +data "archive_file" "fcvm_fleet_on_demand_notify" { + type = "zip" + output_path = "${path.module}/.terraform/fcvm-fleet-on-demand-notify.zip" + source { + content = file("${path.module}/scripts/fcvm-fleet-on-demand-notify.py") + filename = "notify.py" + } +} + +resource "aws_cloudwatch_log_group" "fcvm_fleet_on_demand_notify" { + provider = aws.fcvm_fleet + name = "/aws/lambda/fcvm-fleet-on-demand-notify" + retention_in_days = 30 +} + +resource "aws_iam_role" "fcvm_fleet_on_demand_notify" { + provider = aws.fcvm_fleet + name = "fcvm-fleet-on-demand-notify" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ Effect = "Allow", Principal = { Service = "lambda.amazonaws.com" }, Action = "sts:AssumeRole" }] + }) +} + +resource "aws_iam_role_policy" "fcvm_fleet_on_demand_notify" { + provider = aws.fcvm_fleet + name = "notify-existing-fleet-subscribers" + role = aws_iam_role.fcvm_fleet_on_demand_notify.id + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = ["logs:CreateLogStream", "logs:PutLogEvents"] + Resource = "${aws_cloudwatch_log_group.fcvm_fleet_on_demand_notify.arn}:*" + }, + { + Effect = "Allow" + Action = "sns:Publish" + Resource = local.fcvm_fleet_alert_topic_arns + }, + ] + }) +} + +resource "aws_lambda_function" "fcvm_fleet_on_demand_notify" { + provider = aws.fcvm_fleet + function_name = "fcvm-fleet-on-demand-notify" + filename = data.archive_file.fcvm_fleet_on_demand_notify.output_path + source_code_hash = data.archive_file.fcvm_fleet_on_demand_notify.output_base64sha256 + role = aws_iam_role.fcvm_fleet_on_demand_notify.arn + runtime = "python3.12" + handler = "notify.handler" + timeout = 30 + memory_size = 128 + environment { + variables = { + ALARM_ARN = local.fcvm_fleet_on_demand_alarm_arn + TOPIC_ARNS = jsonencode(local.fcvm_fleet_alert_topic_arns) + } + } + depends_on = [aws_iam_role_policy.fcvm_fleet_on_demand_notify, aws_cloudwatch_log_group.fcvm_fleet_on_demand_notify] +} + +resource "aws_lambda_permission" "fcvm_fleet_on_demand_notify" { + provider = aws.fcvm_fleet + statement_id = "ThisOnDemandLaunchAlarmOnly" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.fcvm_fleet_on_demand_notify.function_name + principal = "lambda.alarms.cloudwatch.amazonaws.com" + source_account = local.fcvm_fleet_account_id + source_arn = local.fcvm_fleet_on_demand_alarm_arn +} + resource "aws_iam_role" "fcvm_fleet_coordinator_execution" { provider = aws.fcvm_fleet name = "fcvm-fleet-coordinator-execution" @@ -614,15 +731,37 @@ resource "aws_iam_role_policy" "fcvm_fleet_coordinator_task" { Condition = { ArnLike = { "ec2:LaunchTemplate" = local.fcvm_fleet_tile_template_condition } } }, { - # GPU hosts (fcvm fleet coord --host-template, ejc3/fcvm#1257): an instant EC2 Fleet over the host templates, spot - # first and on demand when spot is refused. EC2 Fleet launches with this role's RunInstances, so the two statements - # below hold it to the host templates and their own resources as the tile statements above do. + # RunInstances checks a tagged persistent spot request separately and that resource has no LaunchTemplate key. + # The instance/AMI/subnet/interface still need their template-scoped grants; this cannot authorize a launch alone. + Sid = "LaunchTaggedTileSpotRequestsOnly" + Effect = "Allow" + Action = "ec2:RunInstances" + Resource = local.fcvm_fleet_tile_spot_request_arns + Condition = { StringEquals = { + "aws:RequestedRegion" = local.fcvm_fleet_regions + "aws:RequestTag/Purpose" = "fcvm-fleet-tile" + } } + }, + { + # CreateFleet has its own resource authorization, in addition to RunInstances for the resources in a template. + # The fleet itself has no instance-type or instance-tag context; do not require those on its ARN. Sid = "LaunchHostsAsInstantFleets" Effect = "Allow" Action = "ec2:CreateFleet" - Resource = "*" + Resource = concat(local.fcvm_fleet_host_fleet_arns, local.fcvm_fleet_host_volume_arns) Condition = { StringEquals = { "aws:RequestedRegion" = local.fcvm_fleet_regions } } }, + { + Sid = "LaunchFleetInstancesAsTaggedHostsOnly" + Effect = "Allow" + Action = "ec2:CreateFleet" + Resource = local.fcvm_fleet_host_instance_arns + Condition = { StringEquals = { + "aws:RequestedRegion" = local.fcvm_fleet_regions + "ec2:InstanceType" = local.fcvm_fleet_host_types + "aws:RequestTag/Purpose" = "fcvm-fleet-host" + } } + }, { Sid = "LaunchHostsFromTheirTemplates" Effect = "Allow" @@ -631,19 +770,24 @@ resource "aws_iam_role_policy" "fcvm_fleet_coordinator_task" { Condition = { ArnLike = { "ec2:LaunchTemplate" = local.fcvm_fleet_host_template_condition } } }, { - # The host AMIs, the fleet's subnets and group, and the host templates. - Sid = "LaunchHostsWithTheirOwnResources" - Effect = "Allow" - Action = "ec2:RunInstances" - Resource = local.fcvm_fleet_host_referenced_arns - Condition = { ArnLike = { "ec2:LaunchTemplate" = local.fcvm_fleet_host_template_condition } } + # Exactly the host AMIs, host-eligible subnets, group and templates for both authorization actions. RunInstances + # instance creation above still requires a host template. IfExists preserves the template association when + # RunInstances supplies that key, while CreateFleet authorizes the exact template/resource ARNs without the key. + Sid = "LaunchHostsWithTheirOwnResources" + Effect = "Allow" + Action = ["ec2:RunInstances", "ec2:CreateFleet"] + Resource = local.fcvm_fleet_host_referenced_arns + Condition = { + StringEquals = { "aws:RequestedRegion" = local.fcvm_fleet_regions } + ArnLikeIfExists = { "ec2:LaunchTemplate" = local.fcvm_fleet_host_template_condition } + } }, { # A launch from a template can still override the instance type: only the fleet's four types, whatever the # request says, and each template only its own kind (below). Sid = "FleetTypesOnly" Effect = "Deny" - Action = "ec2:RunInstances" + Action = ["ec2:RunInstances", "ec2:CreateFleet"] Resource = "arn:aws:ec2:*:*:instance/*" Condition = { StringNotEquals = { "ec2:InstanceType" = concat(local.fcvm_fleet_tile_types, local.fcvm_fleet_host_types) } } }, @@ -672,7 +816,7 @@ resource "aws_iam_role_policy" "fcvm_fleet_coordinator_task" { # is limited to. Sid = "FleetInstancesAreTagged" Effect = "Deny" - Action = "ec2:RunInstances" + Action = ["ec2:RunInstances", "ec2:CreateFleet"] Resource = "arn:aws:ec2:*:*:instance/*" Condition = { StringNotEquals = { "aws:RequestTag/Purpose" = ["fcvm-fleet-tile", "fcvm-fleet-host"] } } }, diff --git a/modules/fcvm-fleet-region/main.tf b/modules/fcvm-fleet-region/main.tf index b2792d57..2f28c992 100644 --- a/modules/fcvm-fleet-region/main.tf +++ b/modules/fcvm-fleet-region/main.tf @@ -1,7 +1,7 @@ # modules/fcvm-fleet-region # # One region of fcvm's GPU VM wall (fcvm-fleet.tf; design ejc3/fcvm#1136, docs/fleet/ARCHITECTURE.md), inside the fcvm-fleet -# member account (fcvm-fleet-account.tf): the network, the GPU host group and the EC2 comparison tiles' launch templates. +# member account (fcvm-fleet-account.tf): the network and the GPU host and EC2 comparison tile launch templates. # # - Network: a VPC with an Amazon-provided IPv6 /56, one public subnet per AZ (IPv4 /20 and IPv6 /64), an internet gateway, # and one security group that admits TCP 443 from anywhere and nothing else. No SSH: hosts are reached with SSM Session @@ -48,10 +48,15 @@ variable "tile_amis" { } variable "hosts_enabled" { - description = "Run one spot GPU host here (fcvm-fleet.tf's local.fcvm_fleet_hosts_enabled); false keeps the group at 0" + description = "Give the coordinator this region's host templates (fcvm-fleet.tf's local.fcvm_fleet_hosts_enabled)" type = bool } +variable "alert_email" { + description = "The existing fleet-alerts subscriber; preserve the topic and confirmed subscription when groups go away" + type = string +} + variable "host_agent_args" { description = "The host agent's arguments (fcvm fleet agent, FLEET_AGENT_ARGS in /etc/fcvm-fleet/agent.env); empty leaves the agent off" type = string @@ -277,6 +282,20 @@ resource "aws_launch_template" "host" { tags = { Name = "fcvm-fleet-host-${each.key}" } } +# Preserve the existing topic and confirmed email subscription at their original Terraform addresses. The coordinator's +# accepted-launch alarm replaces the obsolete Auto Scaling metric; a narrowly scoped Lambda publishes here through SNS. +resource "aws_sns_topic" "alerts" { + count = length(local.host_archs) > 0 ? 1 : 0 + name = "fcvm-fleet-alerts" +} + +resource "aws_sns_topic_subscription" "alerts_email" { + count = length(local.host_archs) > 0 ? 1 : 0 + topic_arn = aws_sns_topic.alerts[0].arn + protocol = "email" + endpoint = var.alert_email +} + # ------------------------------------------------------------------------------------------------ EC2 tiles resource "aws_launch_template" "tile" { for_each = toset(local.tile_archs) @@ -386,7 +405,12 @@ output "host_subnet_ids" { # The coordinator's RunInstances (through CreateFleet) is limited to these templates and the host AMIs. output "host_template_arns" { - value = [for t in aws_launch_template.host : t.arn] + value = local.hosts_on ? [for t in aws_launch_template.host : t.arn] : [] +} + +output "alert_topic_arns" { + description = "Existing fleet alert destinations, with their confirmed subscriptions retained" + value = aws_sns_topic.alerts[*].arn } output "host_types" { diff --git a/scripts/fcvm-fleet-on-demand-notify.py b/scripts/fcvm-fleet-on-demand-notify.py new file mode 100644 index 00000000..6f5b8e93 --- /dev/null +++ b/scripts/fcvm-fleet-on-demand-notify.py @@ -0,0 +1,22 @@ +"""Forward one accepted-on-demand-launch alarm to the fleet's existing SNS subscribers.""" +import json +import os + +import boto3 + + +def handler(event, context): + state = event.get("alarmData", {}).get("state", {}) + if (event.get("source") != "aws.cloudwatch" + or event.get("alarmArn") != os.environ["ALARM_ARN"] + or state.get("value") != "ALARM"): + return {"notified": 0} + topics = json.loads(os.environ["TOPIC_ARNS"]) + message = ("The coordinator accepted an on-demand GPU host launch. " + "A g4dn.metal costs $7.824/hour in us-east-1 and runs until the fleet sleeps. " + "Inspect the coordinator hosts and logs; this is a launch event, not a running-host count.\n\n" + + json.dumps(event, sort_keys=True)) + for topic in topics: + boto3.client("sns", region_name=topic.split(":")[3]).publish( + TopicArn=topic, Subject="fcvm fleet: on-demand GPU host launched", Message=message) + return {"notified": len(topics)} diff --git a/scripts/test-fcvm-fleet.py b/scripts/test-fcvm-fleet.py index 256aa1dc..aa0da2c6 100644 --- a/scripts/test-fcvm-fleet.py +++ b/scripts/test-fcvm-fleet.py @@ -3,8 +3,11 @@ the GPU hosts and the coordinator stay off until their gates and inputs exist, and the coordinator's role reaches GPU hosts and EC2 tiles from their templates only.""" import re +import importlib.util +import json import unittest from pathlib import Path +from unittest import mock ROOT = Path(__file__).resolve().parent.parent TF = (ROOT / "fcvm-fleet.tf").read_text() @@ -117,6 +120,71 @@ def test_the_host_templates_reach_the_coordinator_in_its_environment(self): for gone in ("--scale-group", "--on-demand-group", "--spot-wait", "--host-template"): self.assertNotIn(gone, cmd) + def test_accepted_on_demand_launches_keep_the_existing_notification_destination(self): + # Preserve SNS addresses so the confirmed subscriber is not deleted/recreated with the Auto Scaling groups. + topic = block('resource "aws_sns_topic" "alerts"', MOD) + subscriber = block('resource "aws_sns_topic_subscription" "alerts_email"', MOD) + self.assertIn('name = "fcvm-fleet-alerts"', topic) + self.assertIn('topic_arn = aws_sns_topic.alerts[0].arn', subscriber) + self.assertIn('endpoint = var.alert_email', subscriber) + metric = block('resource "aws_cloudwatch_log_metric_filter" "fcvm_fleet_on_demand_launches"') + self.assertIn('pattern = "\\\"wake: host launch accepted\\\" \\\"on-demand\\\""', metric) + self.assertNotIn('spot launch refused', metric) + alarm = block('resource "aws_cloudwatch_metric_alarm" "fcvm_fleet_on_demand_launches"') + self.assertIn('statistic = "Sum"', alarm) + self.assertIn('threshold = 1', alarm) + self.assertIn('alarm_actions = [aws_lambda_function.fcvm_fleet_on_demand_notify.arn]', alarm) + for region in ('east2', 'west2', 'east1'): + self.assertIn('module.fcvm_fleet_%s.alert_topic_arns' % region, TF) + # A launch-event counter returning to zero says nothing about whether the host is still charging. + self.assertNotIn('ok_actions', alarm) + permission = block('resource "aws_lambda_permission" "fcvm_fleet_on_demand_notify"') + self.assertIn('principal = "lambda.alarms.cloudwatch.amazonaws.com"', permission) + self.assertIn('source_arn = local.fcvm_fleet_on_demand_alarm_arn', permission) + self.assertIn('source_account = local.fcvm_fleet_account_id', permission) + policy = block('resource "aws_iam_role_policy" "fcvm_fleet_on_demand_notify"') + self.assertIn('Action = "sns:Publish"', policy) + self.assertIn('Resource = local.fcvm_fleet_alert_topic_arns', policy) + self.assertNotIn('Resource = "*"', policy) + + +class OnDemandNotificationTests(unittest.TestCase): + ALARM = 'arn:aws:cloudwatch:us-east-2:123456789012:alarm:fcvm-fleet-on-demand-host-launches' + TOPICS = ['arn:aws:sns:us-east-1:123456789012:fcvm-fleet-alerts', + 'arn:aws:sns:us-west-2:123456789012:fcvm-fleet-alerts'] + + def setUp(self): + self.sdk = mock.Mock() + spec = importlib.util.spec_from_file_location('fleet_notify', ROOT / 'scripts/fcvm-fleet-on-demand-notify.py') + self.notify = importlib.util.module_from_spec(spec) + with mock.patch.dict('sys.modules', {'boto3': self.sdk}): + spec.loader.exec_module(self.notify) + env = mock.patch.dict('os.environ', {'ALARM_ARN': self.ALARM, 'TOPIC_ARNS': json.dumps(self.TOPICS)}) + env.start() + self.addCleanup(env.stop) + + def event(self, state='ALARM', alarm=None, source='aws.cloudwatch'): + return {'source': source, 'alarmArn': alarm or self.ALARM, + 'alarmData': {'state': {'value': state, 'reason': 'accepted on-demand launch'}}} + + def test_accepted_launch_alarm_uses_each_existing_topics_regional_endpoint(self): + self.assertEqual(self.notify.handler(self.event(), None), {'notified': 2}) + self.assertEqual(self.sdk.client.call_args_list, + [mock.call('sns', region_name='us-east-1'), mock.call('sns', region_name='us-west-2')]) + calls = self.sdk.client.return_value.publish.call_args_list + self.assertEqual([c.kwargs['TopicArn'] for c in calls], self.TOPICS) + self.assertTrue(all('not a running-host count' in c.kwargs['Message'] for c in calls)) + + def test_ok_other_alarms_and_other_event_sources_do_not_send(self): + for event in (self.event(state='OK'), self.event(alarm='unrelated'), self.event(source='unrelated'), {}): + self.assertEqual(self.notify.handler(event, None), {'notified': 0}) + self.sdk.client.assert_not_called() + + def test_publish_failures_raise_for_lambda_asynchronous_retry(self): + self.sdk.client.return_value.publish.side_effect = RuntimeError('publish unavailable') + with self.assertRaisesRegex(RuntimeError, 'publish unavailable'): + self.notify.handler(self.event(), None) + class CoordinatorTests(unittest.TestCase): def test_the_service_runs_the_tagged_image_from_the_fleet_repository(self): @@ -180,7 +248,12 @@ def test_created_and_referenced_resources_are_split_and_pinned(self): refs = self.TF.split("fcvm_fleet_tile_referenced_arns = concat(", 1)[1].split("\n )\n", 1)[0] for part in (":image/${ami}", ":subnet/${s}", ":security-group/${m.security_group_id}", "local.fcvm_fleet_tile_template_arns"): self.assertIn(part, refs, part) - self.assertIn('"instance", "volume", "network-interface", "spot-instances-request"', self.TF) + self.assertIn('"instance", "volume", "network-interface"', self.TF) + spot = self.stmt("LaunchTaggedTileSpotRequestsOnly") + self.assertIn('Resource = local.fcvm_fleet_tile_spot_request_arns', spot) + self.assertIn('"aws:RequestTag/Purpose" = "fcvm-fleet-tile"', spot) + self.assertIn('"aws:RequestedRegion"', spot) + self.assertNotIn('"ec2:LaunchTemplate"', spot) def test_state_object_secrets_and_spot_role(self): self.assertIn('Resource = "${aws_s3_bucket.fcvm_fleet_coordinator_state.arn}/state.json"', self.TF) @@ -204,21 +277,39 @@ def test_the_coordinator_launches_hosts_only_from_their_templates(self): fleet = self.stmt("LaunchHostsAsInstantFleets") self.assertIn('Action = "ec2:CreateFleet"', fleet) self.assertIn('"aws:RequestedRegion" = local.fcvm_fleet_regions', fleet) + self.assertIn('Resource = concat(local.fcvm_fleet_host_fleet_arns, local.fcvm_fleet_host_volume_arns)', fleet) + self.assertNotIn('Resource = "*"', fleet) + instances = self.stmt("LaunchFleetInstancesAsTaggedHostsOnly") + self.assertIn('Resource = local.fcvm_fleet_host_instance_arns', instances) + self.assertIn('"aws:RequestTag/Purpose" = "fcvm-fleet-host"', instances) + self.assertIn('"ec2:InstanceType"', instances) + fleet_refs = self.stmt("LaunchHostsWithTheirOwnResources") + self.assertRegex(fleet_refs, r'Resource\s*=\s*local\.fcvm_fleet_host_referenced_arns') + self.assertRegex(fleet_refs, r'Action\s*=\s*\["ec2:RunInstances", "ec2:CreateFleet"\]') + # Instance context is absent on fleet, volume, AMI, template and subnet authorization checks. + for s in (fleet, fleet_refs): + self.assertNotIn('"ec2:InstanceType"', s) + self.assertNotIn('"aws:RequestTag/Purpose"', s) created = self.stmt("LaunchHostsFromTheirTemplates") referenced = self.stmt("LaunchHostsWithTheirOwnResources") - self.assertIn("Resource = local.fcvm_fleet_host_referenced_arns", referenced) - for s in (created, referenced): - self.assertIn('"ec2:LaunchTemplate" = local.fcvm_fleet_host_template_condition', s) + self.assertRegex(referenced, r'Resource\s*=\s*local\.fcvm_fleet_host_referenced_arns') + self.assertIn('"ec2:LaunchTemplate" = local.fcvm_fleet_host_template_condition', created) + self.assertIn('"aws:RequestedRegion" = local.fcvm_fleet_regions', referenced) + self.assertIn('ArnLikeIfExists = { "ec2:LaunchTemplate" = local.fcvm_fleet_host_template_condition }', referenced) refs = self.TF.split("fcvm_fleet_host_referenced_arns = concat(", 1)[1].split("\n )\n", 1)[0] for part in ("var.fcvm_fleet_host_amis", ":subnet/${s}", ":security-group/${m.security_group_id}", "local.fcvm_fleet_host_template_arns"): self.assertIn(part, refs, part) - # A host template launches host types only, and EC2 Fleet's service-linked role exists. + self.assertIn('m.host_subnet_ids', refs) + self.assertIn('if length(m.host_template_names) > 0', refs) + self.assertIn('value = local.hosts_on ? [for t in aws_launch_template.host : t.arn] : []', + block('output "host_template_arns"', MOD)) + # A host template launches host types only. Instant fleets need no EC2 Fleet service-linked role. deny = self.stmt("HostTemplatesLaunchHostTypesOnly") self.assertIn('Effect = "Deny"', deny) self.assertIn('ArnLike = { "ec2:LaunchTemplate" = local.fcvm_fleet_host_template_condition }', deny) self.assertIn('StringNotEquals = { "ec2:InstanceType" = local.fcvm_fleet_host_types }', deny) self.assertIn('fcvm_fleet_host_types = ["g4dn.metal", "g5g.metal"]', self.TF) - self.assertIn('aws_service_name = "ec2fleet.amazonaws.com"', self.TF) + self.assertNotIn('aws_service_name = "ec2fleet.amazonaws.com"', self.TF) self.assertNotIn("autoscaling:", self.TF) def test_an_on_demand_tile_template_is_a_launchable_template(self): @@ -249,8 +340,10 @@ def test_the_coordinator_launches_only_the_fleet_types_and_tile_templates_only_t stmt = CoordinatorLaunchSplit.stmt s = stmt(CoordinatorLaunchSplit, "FleetTypesOnly") self.assertIn('Effect = "Deny"', s) - self.assertIn('Action = "ec2:RunInstances"', s) + self.assertIn('Action = ["ec2:RunInstances", "ec2:CreateFleet"]', s) self.assertIn('"ec2:InstanceType" = concat(local.fcvm_fleet_tile_types, local.fcvm_fleet_host_types)', s) + tagged = stmt(CoordinatorLaunchSplit, "FleetInstancesAreTagged") + self.assertIn('Action = ["ec2:RunInstances", "ec2:CreateFleet"]', tagged) t = stmt(CoordinatorLaunchSplit, "TileTemplatesLaunchTileTypesOnly") self.assertIn('ArnLike = { "ec2:LaunchTemplate" = local.fcvm_fleet_tile_template_condition }', t) self.assertIn('StringNotEquals = { "ec2:InstanceType" = local.fcvm_fleet_tile_types }', t) From dd44218d1e3aa9d938203588e44196646e016339 Mon Sep 17 00:00:00 2001 From: ejc3 Date: Mon, 26 Oct 2026 02:55:49 +0000 Subject: [PATCH 5/5] Remove duplicate pool grants introduced by merge resolution --- fcvm-fleet.tf | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/fcvm-fleet.tf b/fcvm-fleet.tf index 1139f7c5..6a88a7bf 100644 --- a/fcvm-fleet.tf +++ b/fcvm-fleet.tf @@ -868,34 +868,6 @@ resource "aws_iam_role_policy" "fcvm_fleet_coordinator_task" { Resource = "arn:aws:ec2:*:*:spot-instances-request/*" Condition = { StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } } }, - { - # The tile pool (ejc3/fcvm#1273): stop a tile into the pool when the fleet sleeps and start it at the next wake. - Sid = "StartAndStopTilesOnly" - Effect = "Allow" - Action = ["ec2:StartInstances", "ec2:StopInstances"] - Resource = "arn:aws:ec2:*:*:instance/*" - Condition = { StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } } - }, - { - # Moving a tile into and out of the pool retags it (FleetPool, FleetTile) and nothing else. - Sid = "RetagTilesForThePool" - Effect = "Allow" - Action = ["ec2:CreateTags", "ec2:DeleteTags"] - Resource = "arn:aws:ec2:*:*:instance/*" - Condition = { - StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } - "ForAllValues:StringEquals" = { "aws:TagKeys" = ["FleetPool", "FleetTile"] } - } - }, - { - # A pool tile's spot request is persistent: it is cancelled before its instance is terminated, or it launches a - # replacement. The pool template tags its requests Purpose=fcvm-fleet-tile. - Sid = "CancelTileSpotRequestsOnly" - Effect = "Allow" - Action = "ec2:CancelSpotInstanceRequests" - Resource = "arn:aws:ec2:*:*:spot-instances-request/*" - Condition = { StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } } - }, { Sid = "DescribeEc2" Effect = "Allow"