fcvm-fleet: a pool of stopped EC2 tiles the coordinator starts at a wake

fcvm-fleet.tf

@@ -53,6 +53,11 @@ locals {
   fcvm_fleet_coordinator_tag = "bdaa0003"
   # Most on-demand GPU hosts per region (the fallback group's max_size; owner, 2026-10-24: "Up to 8 on-demand max").
   fcvm_fleet_on_demand_max = 8
+  # Whether the coordinator keeps its 8 EC2 tiles stopped between wakes and starts them at the next one (--ec2-pool,
+  # ejc3/fcvm#1273) instead of launching them: a tile then reaches its first frame in about 16 s instead of 81 to 84 s.
+  # Needs a coordinator image with --ec2-pool (an unknown flag stops the task). Standing cost: 8 stopped 32 GB gp3 root
+  # volumes, $20.48 a month.
+  fcvm_fleet_tile_pool = true
   # Whether the coordinator gives a short spot group's capacity to the region's on-demand group (--on-demand-group,
   # --spot-wait, ejc3/fcvm#1215). On with the image at c93a01d9 (ejc3/fcvm main, which has those flags); the image at
   # 8c96e28e refuses an unknown argument and would not start, so the two change together.
@@ -124,6 +129,7 @@ locals {
       ["--spot-wait", "120"],
     ) : [],
     ["--ec2", "aws", "--ec2-tiles", "8", "--ec2-regions", "us-east-1", "--ec2-template", "fcvm-fleet-tile-x86_64", "--ec2-on-demand-template", "fcvm-fleet-tile-on-demand-x86_64"],
+    local.fcvm_fleet_tile_pool ? ["--ec2-pool", "8", "--ec2-pool-template", "fcvm-fleet-tile-pool-x86_64"] : [],
     ["--dns-zone", "fcvm.dev", "--dns-token-secret", aws_secretsmanager_secret.fcvm_fleet_dns_token.name, "--dns-zone-id", var.fcvm_dev_zone_id, "--tls-secret", aws_secretsmanager_secret.fcvm_fleet_tls.name, "--secrets-region", local.fcvm_fleet_home_region],
     # Launch names are served by the coordinator itself, from the subzone fcvm-fleet-dns.tf delegates to it.
     ["--dns-subzone", local.fcvm_fleet_names_zone, "--dns-subzone-ns", join(",", local.fcvm_fleet_names_ns), "--dns-listen", "0.0.0.0:${local.fcvm_fleet_dns_port}"],
@@ -435,11 +441,13 @@ locals {
     local.fcvm_fleet_tile_template_arns,
     ["arn:aws:ec2:us-east-1::image/ami-none"],
   )
-  # What a tile launch creates: the instance, its volume, its network interface and its one-time spot request.
+  # Created resources with launch-template context. A tagged spot request has no ec2:LaunchTemplate context and gets
+  # its own region/Purpose-scoped authorization below; putting that condition on its ARN rejects persistent tile pools.
   fcvm_fleet_tile_created_arns = flatten([for r in local.fcvm_fleet_regions : [
-    for kind in ["instance", "volume", "network-interface", "spot-instances-request"] :
+    for kind in ["instance", "volume", "network-interface"] :
     "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:${kind}/*"
   ]])
+  fcvm_fleet_tile_spot_request_arns  = [for r in local.fcvm_fleet_regions : "arn:aws:ec2:${r}:${local.fcvm_fleet_account_id}:spot-instances-request/*"]
   fcvm_fleet_tile_template_condition = length(local.fcvm_fleet_tile_template_arns) > 0 ? local.fcvm_fleet_tile_template_arns : ["arn:aws:ec2:*:*:launch-template/none"]
 }
 
@@ -619,6 +627,18 @@ resource "aws_iam_role_policy" "fcvm_fleet_coordinator_task" {
         Resource  = local.fcvm_fleet_tile_referenced_arns
         Condition = { ArnLike = { "ec2:LaunchTemplate" = local.fcvm_fleet_tile_template_condition } }
       },
+      {
+        # RunInstances checks a tagged persistent spot request separately and that resource has no LaunchTemplate key.
+        # The instance/AMI/subnet/interface still need their template-scoped grants; this cannot authorize a launch alone.
+        Sid      = "LaunchTaggedTileSpotRequestsOnly"
+        Effect   = "Allow"
+        Action   = "ec2:RunInstances"
+        Resource = local.fcvm_fleet_tile_spot_request_arns
+        Condition = { StringEquals = {
+          "aws:RequestedRegion"    = local.fcvm_fleet_regions
+          "aws:RequestTag/Purpose" = "fcvm-fleet-tile"
+        } }
+      },
       {
         # A launch from a template can still override the instance type: only the two tile types, whatever the request says.
         Sid       = "TileTypesOnly"
@@ -649,6 +669,34 @@ resource "aws_iam_role_policy" "fcvm_fleet_coordinator_task" {
         Resource  = "arn:aws:ec2:*:*:instance/*"
         Condition = { StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } }
       },
+      {
+        # The tile pool (ejc3/fcvm#1273): stop a tile into the pool when the fleet sleeps and start it at the next wake.
+        Sid       = "StartAndStopTilesOnly"
+        Effect    = "Allow"
+        Action    = ["ec2:StartInstances", "ec2:StopInstances"]
+        Resource  = "arn:aws:ec2:*:*:instance/*"
+        Condition = { StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } }
+      },
+      {
+        # Moving a tile into and out of the pool retags it (FleetPool, FleetTile) and nothing else.
+        Sid      = "RetagTilesForThePool"
+        Effect   = "Allow"
+        Action   = ["ec2:CreateTags", "ec2:DeleteTags"]
+        Resource = "arn:aws:ec2:*:*:instance/*"
+        Condition = {
+          StringEquals                = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions }
+          "ForAllValues:StringEquals" = { "aws:TagKeys" = ["FleetPool", "FleetTile"] }
+        }
+      },
+      {
+        # A pool tile's spot request is persistent: it is cancelled before its instance is terminated, or it launches a
+        # replacement. The pool template tags its requests Purpose=fcvm-fleet-tile.
+        Sid       = "CancelTileSpotRequestsOnly"
+        Effect    = "Allow"
+        Action    = "ec2:CancelSpotInstanceRequests"
+        Resource  = "arn:aws:ec2:*:*:spot-instances-request/*"
+        Condition = { StringEquals = { "ec2:ResourceTag/Purpose" = "fcvm-fleet-tile", "aws:RequestedRegion" = local.fcvm_fleet_regions } }
+      },
       {
         Sid      = "DescribeEc2"
         Effect   = "Allow"

modules/fcvm-fleet-region/main.tf

@@ -518,6 +518,59 @@ resource "aws_launch_template" "tile_on_demand" {
   tags = { Name = "fcvm-fleet-tile-on-demand-${each.key}" }
 }
 
+# The tile the coordinator keeps a pool of (fcvm fleet coord --ec2-pool <n> --ec2-pool-template
+# fcvm-fleet-tile-pool-<arch>, ejc3/fcvm#1273): persistent spot that stops on interruption, so the coordinator can stop a
+# tile when the fleet sleeps and start it at the next wake (15.5 s to a first frame, against 81 to 84 s for a launch,
+# g4dn.xlarge, 2026-10-25). The spot request is tagged too, so the coordinator's CancelSpotInstanceRequests (it cancels a
+# persistent request before terminating its instance, or the request launches a replacement) is limited to tiles'.
+# Everything else is the spot template's.
+resource "aws_launch_template" "tile_pool" {
+  for_each      = toset(local.tile_archs)
+  name          = "fcvm-fleet-tile-pool-${each.key}"
+  description   = "fcvm fleet EC2 comparison tile, ${each.key} (${local.tile_types[each.key]}): persistent spot, stopped between wakes"
+  image_id      = var.tile_amis[each.key]
+  instance_type = local.tile_types[each.key]
+
+  iam_instance_profile {
+    name = var.tile_profile_name
+  }
+
+  instance_market_options {
+    market_type = "spot"
+    spot_options {
+      spot_instance_type             = "persistent"
+      instance_interruption_behavior = "stop"
+    }
+  }
+
+  metadata_options {
+    http_endpoint               = "enabled"
+    http_tokens                 = "required"
+    http_put_response_hop_limit = 1
+  }
+
+  network_interfaces {
+    device_index                = 0
+    security_groups             = [aws_security_group.fleet.id]
+    associate_public_ip_address = true
+    ipv6_address_count          = 1
+    delete_on_termination       = true
+    subnet_id                   = aws_subnet.public[local.tile_az].id
+  }
+
+  tag_specifications {
+    resource_type = "instance"
+    tags          = { Name = "fcvm-fleet-tile", Purpose = "fcvm-fleet-tile" }
+  }
+
+  tag_specifications {
+    resource_type = "spot-instances-request"
+    tags          = { Name = "fcvm-fleet-tile", Purpose = "fcvm-fleet-tile" }
+  }
+
+  tags = { Name = "fcvm-fleet-tile-pool-${each.key}" }
+}
+
 output "vpc_id" {
   value = aws_vpc.this.id
 }
@@ -530,9 +583,9 @@ output "security_group_id" {
   value = aws_security_group.fleet.id
 }
 
-# Both markets: the coordinator's RunInstances is limited to these templates.
+# Both markets and the pool: the coordinator's RunInstances is limited to these templates.
 output "tile_template_arns" {
-  value = concat([for t in aws_launch_template.tile : t.arn], [for t in aws_launch_template.tile_on_demand : t.arn])
+  value = concat([for t in aws_launch_template.tile : t.arn], [for t in aws_launch_template.tile_on_demand : t.arn], [for t in aws_launch_template.tile_pool : t.arn])
 }
 
 output "host_group_desired" {

scripts/test-fcvm-fleet.py

@@ -213,7 +213,12 @@ def test_created_and_referenced_resources_are_split_and_pinned(self):
         refs = self.TF.split("fcvm_fleet_tile_referenced_arns = concat(", 1)[1].split("\n  )\n", 1)[0]
         for part in (":image/${ami}", ":subnet/${s}", ":security-group/${m.security_group_id}", "local.fcvm_fleet_tile_template_arns"):
             self.assertIn(part, refs, part)
-        self.assertIn('"instance", "volume", "network-interface", "spot-instances-request"', self.TF)
+        self.assertIn('"instance", "volume", "network-interface"', self.TF)
+        spot = self.stmt("LaunchTaggedTileSpotRequestsOnly")
+        self.assertIn('Resource = local.fcvm_fleet_tile_spot_request_arns', spot)
+        self.assertIn('"aws:RequestTag/Purpose" = "fcvm-fleet-tile"', spot)
+        self.assertIn('"aws:RequestedRegion"', spot)
+        self.assertNotIn('"ec2:LaunchTemplate"', spot)
 
     def test_state_object_secrets_and_spot_role(self):
         self.assertIn('Resource = "${aws_s3_bucket.fcvm_fleet_coordinator_state.arn}/state.json"', self.TF)
@@ -258,6 +263,35 @@ def test_an_on_demand_tile_template_is_a_launchable_template(self):
         self.assertIn("[for t in aws_launch_template.tile_on_demand : t.arn]", block('output "tile_template_arns"', MOD))
 
 
+class TilePool(unittest.TestCase):
+    TF = CoordinatorLaunchSplit.TF
+    stmt = CoordinatorLaunchSplit.stmt
+
+    def test_the_pool_template_is_persistent_spot_that_stops_and_tags_its_request(self):
+        pool = block('resource "aws_launch_template" "tile_pool"', MOD)
+        self.assertIn('spot_instance_type             = "persistent"', pool)
+        self.assertIn('instance_interruption_behavior = "stop"', pool)
+        self.assertIn('resource_type = "spot-instances-request"', pool)
+        self.assertEqual(pool.count('Purpose = "fcvm-fleet-tile"'), 2)
+        self.assertIn("[for t in aws_launch_template.tile_pool : t.arn]", block('output "tile_template_arns"', MOD))
+
+    def test_start_stop_retag_and_cancel_are_limited_to_tiles(self):
+        ss = self.stmt("StartAndStopTilesOnly")
+        self.assertIn('["ec2:StartInstances", "ec2:StopInstances"]', ss)
+        self.assertIn('"ec2:ResourceTag/Purpose" = "fcvm-fleet-tile"', ss)
+        rt = self.stmt("RetagTilesForThePool")
+        self.assertIn('"ec2:ResourceTag/Purpose" = "fcvm-fleet-tile"', rt)
+        self.assertIn('"aws:TagKeys" = ["FleetPool", "FleetTile"]', rt)
+        cs = self.stmt("CancelTileSpotRequestsOnly")
+        self.assertIn('Resource  = "arn:aws:ec2:*:*:spot-instances-request/*"', cs)
+        self.assertIn('"ec2:ResourceTag/Purpose" = "fcvm-fleet-tile"', cs)
+        for s in (ss, rt, cs):
+            self.assertNotIn('Resource = "*"', s)
+
+    def test_the_coordinator_gets_the_pool_flags_only_behind_the_switch(self):
+        self.assertIn('local.fcvm_fleet_tile_pool ? ["--ec2-pool", "8", "--ec2-pool-template", "fcvm-fleet-tile-pool-x86_64"] : []', self.TF)
+
+
 class ModuleDependencies(unittest.TestCase):
     def test_no_module_level_depends_on(self):
         # A module-level depends_on defers the module's data sources (the AZ list) to apply time, and the subnets'