diff --git a/fcvm-fleet.tf b/fcvm-fleet.tf index 605b766e..f1e8415b 100644 --- a/fcvm-fleet.tf +++ b/fcvm-fleet.tf @@ -53,6 +53,11 @@ locals { fcvm_fleet_coordinator_tag = "bdaa0003" # Most on-demand GPU hosts per region (the fallback group's max_size; owner, 2026-10-24: "Up to 8 on-demand max"). fcvm_fleet_on_demand_max = 8 + # Whether the coordinator keeps its 8 EC2 tiles stopped between wakes and starts them at the next one (--ec2-pool, + # ejc3/fcvm#1273) instead of launching them: a tile then reaches its first frame in about 16 s instead of 81 to 84 s. + # Needs a coordinator image with --ec2-pool (an unknown flag stops the task). Standing cost: 8 stopped 32 GB gp3 root + # volumes, $20.48 a month. + fcvm_fleet_tile_pool = true # Whether the coordinator gives a short spot group's capacity to the region's on-demand group (--on-demand-group, # --spot-wait, ejc3/fcvm#1215). On with the image at c93a01d9 (ejc3/fcvm main, which has those flags); the image at # 8c96e28e refuses an unknown argument and would not start, so the two change together. @@ -124,6 +129,7 @@ locals { ["--spot-wait", "120"], ) : [], ["--ec2", "aws", "--ec2-tiles", "8", "--ec2-regions", "us-east-1", "--ec2-template", "fcvm-fleet-tile-x86_64", "--ec2-on-demand-template", "fcvm-fleet-tile-on-demand-x86_64"], + local.fcvm_fleet_tile_pool ? ["--ec2-pool", "8", "--ec2-pool-template", "fcvm-fleet-tile-pool-x86_64"] : [], ["--dns-zone", "fcvm.dev", "--dns-token-secret", aws_secretsmanager_secret.fcvm_fleet_dns_token.name, "--dns-zone-id", var.fcvm_dev_zone_id, "--tls-secret", aws_secretsmanager_secret.fcvm_fleet_tls.name, "--secrets-region", local.fcvm_fleet_home_region], # Launch names are served by the coordinator itself, from the subzone fcvm-fleet-dns.tf delegates to it. ["--dns-subzone", local.fcvm_fleet_names_zone, "--dns-subzone-ns", join(",", local.fcvm_fleet_names_ns), "--dns-listen", "0.0.0.0:${local.fcvm_fleet_dns_port}"], @@ -435,11 +441,13 @@ locals { local.fcvm_fleet_tile_template_arns, ["arn:aws:ec2:us-east-1::image/ami-none"], ) - # What a tile launch creates: the instance, its volume, its network interface and its one-time spot request. + # Created resources with launch-template context. A tagged spot request has no ec2:LaunchTemplate context and gets + # its own region/Purpose-scoped authorization below; putting that condition on its ARN rejects persistent tile pools. fcvm_fleet_tile_created_arns = flatten([for r in local.fcvm_fleet_regions : [ - for kind in ["instance", "volume", "network-interface", "spot-instances-request"] : + for kind in ["instance", "volume", "network-interface"] : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:${kind}/*" ]]) + fcvm_fleet_tile_spot_request_arns = [for r in local.fcvm_fleet_regions : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:spot-instances-request/*"] fcvm_fleet_tile_template_condition = length(local.fcvm_fleet_tile_template_arns) > 0 ? local.fcvm_fleet_tile_template_arns : ["arn:aws:ec2:*:*:launch-template/none"] } @@ -619,6 +627,18 @@ resource "aws_iam_role_policy" "fcvm_fleet_coordinator_task" { Resource = local.fcvm_fleet_tile_referenced_arns Condition = { ArnLike = { "ec2:LaunchTemplate" = local.fcvm_fleet_tile_template_condition } } }, + { + # RunInstances checks a tagged persistent spot request separately and that resource has no LaunchTemplate key. + # The instance/AMI/subnet/interface still need their template-scoped grants; this cannot authorize a launch alone. + Sid = "LaunchTaggedTileSpotRequestsOnly" + Effect = "Allow" + Action = "ec2:RunInstances" + Resource = local.fcvm_fleet_tile_spot_request_arns + Condition = { StringEquals = { + "aws:RequestedRegion" = local.fcvm_fleet_regions + "aws:RequestTag/Purpose" = "fcvm-fleet-tile" + } } + }, { # A launch from a template can still override the instance type: only the two tile types, whatever the request says. Sid = "TileTypesOnly" @@ -649,6 +669,34 @@ resource "aws_iam_role_policy" "fcvm_fleet_coordinator_task" { Resource = "arn:aws:ec2:*:*:instance/*" Condition = { StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } } }, + { + # The tile pool (ejc3/fcvm#1273): stop a tile into the pool when the fleet sleeps and start it at the next wake. + Sid = "StartAndStopTilesOnly" + Effect = "Allow" + Action = ["ec2:StartInstances", "ec2:StopInstances"] + Resource = "arn:aws:ec2:*:*:instance/*" + Condition = { StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } } + }, + { + # Moving a tile into and out of the pool retags it (FleetPool, FleetTile) and nothing else. + Sid = "RetagTilesForThePool" + Effect = "Allow" + Action = ["ec2:CreateTags", "ec2:DeleteTags"] + Resource = "arn:aws:ec2:*:*:instance/*" + Condition = { + StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } + "ForAllValues:StringEquals" = { "aws:TagKeys" = ["FleetPool", "FleetTile"] } + } + }, + { + # A pool tile's spot request is persistent: it is cancelled before its instance is terminated, or it launches a + # replacement. The pool template tags its requests Purpose=fcvm-fleet-tile. + Sid = "CancelTileSpotRequestsOnly" + Effect = "Allow" + Action = "ec2:CancelSpotInstanceRequests" + Resource = "arn:aws:ec2:*:*:spot-instances-request/*" + Condition = { StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } } + }, { Sid = "DescribeEc2" Effect = "Allow" diff --git a/modules/fcvm-fleet-region/main.tf b/modules/fcvm-fleet-region/main.tf index b41f82df..34bbe593 100644 --- a/modules/fcvm-fleet-region/main.tf +++ b/modules/fcvm-fleet-region/main.tf @@ -518,6 +518,59 @@ resource "aws_launch_template" "tile_on_demand" { tags = { Name = "fcvm-fleet-tile-on-demand-${each.key}" } } +# The tile the coordinator keeps a pool of (fcvm fleet coord --ec2-pool --ec2-pool-template +# fcvm-fleet-tile-pool-, ejc3/fcvm#1273): persistent spot that stops on interruption, so the coordinator can stop a +# tile when the fleet sleeps and start it at the next wake (15.5 s to a first frame, against 81 to 84 s for a launch, +# g4dn.xlarge, 2026-10-25). The spot request is tagged too, so the coordinator's CancelSpotInstanceRequests (it cancels a +# persistent request before terminating its instance, or the request launches a replacement) is limited to tiles'. +# Everything else is the spot template's. +resource "aws_launch_template" "tile_pool" { + for_each = toset(local.tile_archs) + name = "fcvm-fleet-tile-pool-${each.key}" + description = "fcvm fleet EC2 comparison tile, ${each.key} (${local.tile_types[each.key]}): persistent spot, stopped between wakes" + image_id = var.tile_amis[each.key] + instance_type = local.tile_types[each.key] + + iam_instance_profile { + name = var.tile_profile_name + } + + instance_market_options { + market_type = "spot" + spot_options { + spot_instance_type = "persistent" + instance_interruption_behavior = "stop" + } + } + + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 1 + } + + network_interfaces { + device_index = 0 + security_groups = [aws_security_group.fleet.id] + associate_public_ip_address = true + ipv6_address_count = 1 + delete_on_termination = true + subnet_id = aws_subnet.public[local.tile_az].id + } + + tag_specifications { + resource_type = "instance" + tags = { Name = "fcvm-fleet-tile", Purpose = "fcvm-fleet-tile" } + } + + tag_specifications { + resource_type = "spot-instances-request" + tags = { Name = "fcvm-fleet-tile", Purpose = "fcvm-fleet-tile" } + } + + tags = { Name = "fcvm-fleet-tile-pool-${each.key}" } +} + output "vpc_id" { value = aws_vpc.this.id } @@ -530,9 +583,9 @@ output "security_group_id" { value = aws_security_group.fleet.id } -# Both markets: the coordinator's RunInstances is limited to these templates. +# Both markets and the pool: the coordinator's RunInstances is limited to these templates. output "tile_template_arns" { - value = concat([for t in aws_launch_template.tile : t.arn], [for t in aws_launch_template.tile_on_demand : t.arn]) + value = concat([for t in aws_launch_template.tile : t.arn], [for t in aws_launch_template.tile_on_demand : t.arn], [for t in aws_launch_template.tile_pool : t.arn]) } output "host_group_desired" { diff --git a/scripts/test-fcvm-fleet.py b/scripts/test-fcvm-fleet.py index b6ace6b4..09c3a7ae 100644 --- a/scripts/test-fcvm-fleet.py +++ b/scripts/test-fcvm-fleet.py @@ -213,7 +213,12 @@ def test_created_and_referenced_resources_are_split_and_pinned(self): refs = self.TF.split("fcvm_fleet_tile_referenced_arns = concat(", 1)[1].split("\n )\n", 1)[0] for part in (":image/${ami}", ":subnet/${s}", ":security-group/${m.security_group_id}", "local.fcvm_fleet_tile_template_arns"): self.assertIn(part, refs, part) - self.assertIn('"instance", "volume", "network-interface", "spot-instances-request"', self.TF) + self.assertIn('"instance", "volume", "network-interface"', self.TF) + spot = self.stmt("LaunchTaggedTileSpotRequestsOnly") + self.assertIn('Resource = local.fcvm_fleet_tile_spot_request_arns', spot) + self.assertIn('"aws:RequestTag/Purpose" = "fcvm-fleet-tile"', spot) + self.assertIn('"aws:RequestedRegion"', spot) + self.assertNotIn('"ec2:LaunchTemplate"', spot) def test_state_object_secrets_and_spot_role(self): self.assertIn('Resource = "${aws_s3_bucket.fcvm_fleet_coordinator_state.arn}/state.json"', self.TF) @@ -258,6 +263,35 @@ def test_an_on_demand_tile_template_is_a_launchable_template(self): self.assertIn("[for t in aws_launch_template.tile_on_demand : t.arn]", block('output "tile_template_arns"', MOD)) +class TilePool(unittest.TestCase): + TF = CoordinatorLaunchSplit.TF + stmt = CoordinatorLaunchSplit.stmt + + def test_the_pool_template_is_persistent_spot_that_stops_and_tags_its_request(self): + pool = block('resource "aws_launch_template" "tile_pool"', MOD) + self.assertIn('spot_instance_type = "persistent"', pool) + self.assertIn('instance_interruption_behavior = "stop"', pool) + self.assertIn('resource_type = "spot-instances-request"', pool) + self.assertEqual(pool.count('Purpose = "fcvm-fleet-tile"'), 2) + self.assertIn("[for t in aws_launch_template.tile_pool : t.arn]", block('output "tile_template_arns"', MOD)) + + def test_start_stop_retag_and_cancel_are_limited_to_tiles(self): + ss = self.stmt("StartAndStopTilesOnly") + self.assertIn('["ec2:StartInstances", "ec2:StopInstances"]', ss) + self.assertIn('"ec2:ResourceTag/Purpose" = "fcvm-fleet-tile"', ss) + rt = self.stmt("RetagTilesForThePool") + self.assertIn('"ec2:ResourceTag/Purpose" = "fcvm-fleet-tile"', rt) + self.assertIn('"aws:TagKeys" = ["FleetPool", "FleetTile"]', rt) + cs = self.stmt("CancelTileSpotRequestsOnly") + self.assertIn('Resource = "arn:aws:ec2:*:*:spot-instances-request/*"', cs) + self.assertIn('"ec2:ResourceTag/Purpose" = "fcvm-fleet-tile"', cs) + for s in (ss, rt, cs): + self.assertNotIn('Resource = "*"', s) + + def test_the_coordinator_gets_the_pool_flags_only_behind_the_switch(self): + self.assertIn('local.fcvm_fleet_tile_pool ? ["--ec2-pool", "8", "--ec2-pool-template", "fcvm-fleet-tile-pool-x86_64"] : []', self.TF) + + class ModuleDependencies(unittest.TestCase): def test_no_module_level_depends_on(self): # A module-level depends_on defers the module's data sources (the AZ list) to apply time, and the subnets'