From 10b0cd91797c7bec4a70dd32a8ec8cdbaa9f54ec Mon Sep 17 00:00:00 2001 From: ejc3 Date: Sun, 25 Oct 2026 14:01:59 +0000 Subject: [PATCH 1/2] fcvm-fleet: a pool of stopped EC2 tiles the coordinator starts at a wake The coordinator in ejc3/fcvm#1273 keeps its EC2 tiles stopped between wakes (--ec2-pool) and starts them at the next one instead of launching: on g4dn.xlarge from the tile AMI a stopped tile reached its first frame in 15.5 s (persistent spot) against 81 to 84 s for a launch (2026-10-25). g4dn has no hibernation and Auto Scaling warm pools take no spot. - modules/fcvm-fleet-region: launch template fcvm-fleet-tile-pool-, the spot tile with a persistent request that stops on interruption, its spot request tagged Purpose=fcvm-fleet-tile; in tile_template_arns, so RunInstances may use it. - The coordinator's role: StartInstances/StopInstances and CreateTags/DeleteTags (keys FleetPool and FleetTile only) on instances tagged Purpose=fcvm-fleet-tile, and CancelSpotInstanceRequests on requests so tagged (a persistent request launches a replacement when its instance is terminated, so the coordinator cancels it first). - Coordinator arguments --ec2-pool 8 --ec2-pool-template fcvm-fleet-tile-pool-x86_64 behind local.fcvm_fleet_tile_pool. Standing cost: 8 stopped 32 GB gp3 root volumes, $20.48 a month; no Elastic IPs. Tested: python3 scripts/test-fcvm-fleet.py (39 tests OK, 3 new), terraform fmt. terraform validate was not run here: this box has 1.16.6 and main.tf pins 1.10.3. --- fcvm-fleet.tf | 34 ++++++++++++++++++ modules/fcvm-fleet-region/main.tf | 57 +++++++++++++++++++++++++++++-- scripts/test-fcvm-fleet.py | 29 ++++++++++++++++ 3 files changed, 118 insertions(+), 2 deletions(-) diff --git a/fcvm-fleet.tf b/fcvm-fleet.tf index b376a211..18a5c77b 100644 --- a/fcvm-fleet.tf +++ b/fcvm-fleet.tf @@ -53,6 +53,11 @@ locals { fcvm_fleet_coordinator_tag = "bdaa0003" # Most on-demand GPU hosts per region (the fallback group's max_size; owner, 2026-10-24: "Up to 8 on-demand max"). fcvm_fleet_on_demand_max = 8 + # Whether the coordinator keeps its 8 EC2 tiles stopped between wakes and starts them at the next one (--ec2-pool, + # ejc3/fcvm#1273) instead of launching them: a tile then reaches its first frame in about 16 s instead of 81 to 84 s. + # Needs a coordinator image with --ec2-pool (an unknown flag stops the task). Standing cost: 8 stopped 32 GB gp3 root + # volumes, $20.48 a month. + fcvm_fleet_tile_pool = true # Whether the coordinator gives a short spot group's capacity to the region's on-demand group (--on-demand-group, # --spot-wait, ejc3/fcvm#1215). On with the image at c93a01d9 (ejc3/fcvm main, which has those flags); the image at # 8c96e28e refuses an unknown argument and would not start, so the two change together. @@ -127,6 +132,7 @@ locals { ["--spot-wait", "120"], ) : [], ["--ec2", "aws", "--ec2-tiles", "8", "--ec2-regions", "us-east-1", "--ec2-template", "fcvm-fleet-tile-x86_64", "--ec2-on-demand-template", "fcvm-fleet-tile-on-demand-x86_64"], + local.fcvm_fleet_tile_pool ? ["--ec2-pool", "8", "--ec2-pool-template", "fcvm-fleet-tile-pool-x86_64"] : [], ["--dns-zone", "fcvm.dev", "--dns-token-secret", aws_secretsmanager_secret.fcvm_fleet_dns_token.name, "--dns-zone-id", var.fcvm_dev_zone_id, "--tls-secret", aws_secretsmanager_secret.fcvm_fleet_tls.name, "--secrets-region", local.fcvm_fleet_home_region, "--dns-nameservers", join(",", local.fcvm_dev_nameservers)], # The EC2 half's aggregator beside the coordinator: its layout on loopback, browsers on 443 of the task's own addresses. local.fcvm_fleet_aggregated ? ["--ec2-aggregate-listen", "127.0.0.1:${local.fcvm_fleet_aggregate_port}", "--ec2-aggregate-tls", "--ec2-aggregate-tls-dir", "/run/fcvm-fleet/tls"] : [], @@ -646,6 +652,34 @@ resource "aws_iam_role_policy" "fcvm_fleet_coordinator_task" { Resource = "arn:aws:ec2:*:*:instance/*" Condition = { StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } } }, + { + # The tile pool (ejc3/fcvm#1273): stop a tile into the pool when the fleet sleeps and start it at the next wake. + Sid = "StartAndStopTilesOnly" + Effect = "Allow" + Action = ["ec2:StartInstances", "ec2:StopInstances"] + Resource = "arn:aws:ec2:*:*:instance/*" + Condition = { StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } } + }, + { + # Moving a tile into and out of the pool retags it (FleetPool, FleetTile) and nothing else. + Sid = "RetagTilesForThePool" + Effect = "Allow" + Action = ["ec2:CreateTags", "ec2:DeleteTags"] + Resource = "arn:aws:ec2:*:*:instance/*" + Condition = { + StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } + "ForAllValues:StringEquals" = { "aws:TagKeys" = ["FleetPool", "FleetTile"] } + } + }, + { + # A pool tile's spot request is persistent: it is cancelled before its instance is terminated, or it launches a + # replacement. The pool template tags its requests Purpose=fcvm-fleet-tile. + Sid = "CancelTileSpotRequestsOnly" + Effect = "Allow" + Action = "ec2:CancelSpotInstanceRequests" + Resource = "arn:aws:ec2:*:*:spot-instances-request/*" + Condition = { StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } } + }, { Sid = "DescribeEc2" Effect = "Allow" diff --git a/modules/fcvm-fleet-region/main.tf b/modules/fcvm-fleet-region/main.tf index 0f1e82c3..7af732ca 100644 --- a/modules/fcvm-fleet-region/main.tf +++ b/modules/fcvm-fleet-region/main.tf @@ -502,6 +502,59 @@ resource "aws_launch_template" "tile_on_demand" { tags = { Name = "fcvm-fleet-tile-on-demand-${each.key}" } } +# The tile the coordinator keeps a pool of (fcvm fleet coord --ec2-pool --ec2-pool-template +# fcvm-fleet-tile-pool-, ejc3/fcvm#1273): persistent spot that stops on interruption, so the coordinator can stop a +# tile when the fleet sleeps and start it at the next wake (15.5 s to a first frame, against 81 to 84 s for a launch, +# g4dn.xlarge, 2026-10-25). The spot request is tagged too, so the coordinator's CancelSpotInstanceRequests (it cancels a +# persistent request before terminating its instance, or the request launches a replacement) is limited to tiles'. +# Everything else is the spot template's. +resource "aws_launch_template" "tile_pool" { + for_each = toset(local.tile_archs) + name = "fcvm-fleet-tile-pool-${each.key}" + description = "fcvm fleet EC2 comparison tile, ${each.key} (${local.tile_types[each.key]}): persistent spot, stopped between wakes" + image_id = var.tile_amis[each.key] + instance_type = local.tile_types[each.key] + + iam_instance_profile { + name = var.tile_profile_name + } + + instance_market_options { + market_type = "spot" + spot_options { + spot_instance_type = "persistent" + instance_interruption_behavior = "stop" + } + } + + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 1 + } + + network_interfaces { + device_index = 0 + security_groups = [aws_security_group.fleet.id] + associate_public_ip_address = true + ipv6_address_count = 1 + delete_on_termination = true + subnet_id = aws_subnet.public[local.tile_az].id + } + + tag_specifications { + resource_type = "instance" + tags = { Name = "fcvm-fleet-tile", Purpose = "fcvm-fleet-tile" } + } + + tag_specifications { + resource_type = "spot-instances-request" + tags = { Name = "fcvm-fleet-tile", Purpose = "fcvm-fleet-tile" } + } + + tags = { Name = "fcvm-fleet-tile-pool-${each.key}" } +} + output "vpc_id" { value = aws_vpc.this.id } @@ -514,9 +567,9 @@ output "security_group_id" { value = aws_security_group.fleet.id } -# Both markets: the coordinator's RunInstances is limited to these templates. +# Both markets and the pool: the coordinator's RunInstances is limited to these templates. output "tile_template_arns" { - value = concat([for t in aws_launch_template.tile : t.arn], [for t in aws_launch_template.tile_on_demand : t.arn]) + value = concat([for t in aws_launch_template.tile : t.arn], [for t in aws_launch_template.tile_on_demand : t.arn], [for t in aws_launch_template.tile_pool : t.arn]) } output "host_group_desired" { diff --git a/scripts/test-fcvm-fleet.py b/scripts/test-fcvm-fleet.py index dc303bb7..c20131e7 100644 --- a/scripts/test-fcvm-fleet.py +++ b/scripts/test-fcvm-fleet.py @@ -242,6 +242,35 @@ def test_an_on_demand_tile_template_is_a_launchable_template(self): self.assertIn("[for t in aws_launch_template.tile_on_demand : t.arn]", block('output "tile_template_arns"', MOD)) +class TilePool(unittest.TestCase): + TF = CoordinatorLaunchSplit.TF + stmt = CoordinatorLaunchSplit.stmt + + def test_the_pool_template_is_persistent_spot_that_stops_and_tags_its_request(self): + pool = block('resource "aws_launch_template" "tile_pool"', MOD) + self.assertIn('spot_instance_type = "persistent"', pool) + self.assertIn('instance_interruption_behavior = "stop"', pool) + self.assertIn('resource_type = "spot-instances-request"', pool) + self.assertEqual(pool.count('Purpose = "fcvm-fleet-tile"'), 2) + self.assertIn("[for t in aws_launch_template.tile_pool : t.arn]", block('output "tile_template_arns"', MOD)) + + def test_start_stop_retag_and_cancel_are_limited_to_tiles(self): + ss = self.stmt("StartAndStopTilesOnly") + self.assertIn('["ec2:StartInstances", "ec2:StopInstances"]', ss) + self.assertIn('"ec2:ResourceTag/Purpose" = "fcvm-fleet-tile"', ss) + rt = self.stmt("RetagTilesForThePool") + self.assertIn('"ec2:ResourceTag/Purpose" = "fcvm-fleet-tile"', rt) + self.assertIn('"aws:TagKeys" = ["FleetPool", "FleetTile"]', rt) + cs = self.stmt("CancelTileSpotRequestsOnly") + self.assertIn('Resource = "arn:aws:ec2:*:*:spot-instances-request/*"', cs) + self.assertIn('"ec2:ResourceTag/Purpose" = "fcvm-fleet-tile"', cs) + for s in (ss, rt, cs): + self.assertNotIn('Resource = "*"', s) + + def test_the_coordinator_gets_the_pool_flags_only_behind_the_switch(self): + self.assertIn('local.fcvm_fleet_tile_pool ? ["--ec2-pool", "8", "--ec2-pool-template", "fcvm-fleet-tile-pool-x86_64"] : []', self.TF) + + class ModuleDependencies(unittest.TestCase): def test_no_module_level_depends_on(self): # A module-level depends_on defers the module's data sources (the AZ list) to apply time, and the subnets' From 721185a5207faff8e125cd646fc84d194ee4c186 Mon Sep 17 00:00:00 2001 From: ejc3 Date: Mon, 26 Oct 2026 02:30:42 +0000 Subject: [PATCH 2/2] fix: authorize tagged tile spot requests with supported context --- fcvm-fleet.tf | 18 ++++++++++++++++-- scripts/test-fcvm-fleet.py | 7 ++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/fcvm-fleet.tf b/fcvm-fleet.tf index 18a5c77b..e57e2c7b 100644 --- a/fcvm-fleet.tf +++ b/fcvm-fleet.tf @@ -442,11 +442,13 @@ locals { local.fcvm_fleet_tile_template_arns, ["arn:aws:ec2:us-east-1::image/ami-none"], ) - # What a tile launch creates: the instance, its volume, its network interface and its one-time spot request. + # Created resources with launch-template context. A tagged spot request has no ec2:LaunchTemplate context and gets + # its own region/Purpose-scoped authorization below; putting that condition on its ARN rejects persistent tile pools. fcvm_fleet_tile_created_arns = flatten([for r in local.fcvm_fleet_regions : [ - for kind in ["instance", "volume", "network-interface", "spot-instances-request"] : + for kind in ["instance", "volume", "network-interface"] : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:${kind}/*" ]]) + fcvm_fleet_tile_spot_request_arns = [for r in local.fcvm_fleet_regions : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:spot-instances-request/*"] fcvm_fleet_tile_template_condition = length(local.fcvm_fleet_tile_template_arns) > 0 ? local.fcvm_fleet_tile_template_arns : ["arn:aws:ec2:*:*:launch-template/none"] } @@ -622,6 +624,18 @@ resource "aws_iam_role_policy" "fcvm_fleet_coordinator_task" { Resource = local.fcvm_fleet_tile_referenced_arns Condition = { ArnLike = { "ec2:LaunchTemplate" = local.fcvm_fleet_tile_template_condition } } }, + { + # RunInstances checks a tagged persistent spot request separately and that resource has no LaunchTemplate key. + # The instance/AMI/subnet/interface still need their template-scoped grants; this cannot authorize a launch alone. + Sid = "LaunchTaggedTileSpotRequestsOnly" + Effect = "Allow" + Action = "ec2:RunInstances" + Resource = local.fcvm_fleet_tile_spot_request_arns + Condition = { StringEquals = { + "aws:RequestedRegion" = local.fcvm_fleet_regions + "aws:RequestTag/Purpose" = "fcvm-fleet-tile" + } } + }, { # A launch from a template can still override the instance type: only the two tile types, whatever the request says. Sid = "TileTypesOnly" diff --git a/scripts/test-fcvm-fleet.py b/scripts/test-fcvm-fleet.py index c20131e7..f3c63885 100644 --- a/scripts/test-fcvm-fleet.py +++ b/scripts/test-fcvm-fleet.py @@ -197,7 +197,12 @@ def test_created_and_referenced_resources_are_split_and_pinned(self): refs = self.TF.split("fcvm_fleet_tile_referenced_arns = concat(", 1)[1].split("\n )\n", 1)[0] for part in (":image/${ami}", ":subnet/${s}", ":security-group/${m.security_group_id}", "local.fcvm_fleet_tile_template_arns"): self.assertIn(part, refs, part) - self.assertIn('"instance", "volume", "network-interface", "spot-instances-request"', self.TF) + self.assertIn('"instance", "volume", "network-interface"', self.TF) + spot = self.stmt("LaunchTaggedTileSpotRequestsOnly") + self.assertIn('Resource = local.fcvm_fleet_tile_spot_request_arns', spot) + self.assertIn('"aws:RequestTag/Purpose" = "fcvm-fleet-tile"', spot) + self.assertIn('"aws:RequestedRegion"', spot) + self.assertNotIn('"ec2:LaunchTemplate"', spot) def test_state_object_secrets_and_spot_role(self): self.assertIn('Resource = "${aws_s3_bucket.fcvm_fleet_coordinator_state.arn}/state.json"', self.TF)