From df5af6fb2cdf9f884f5e71abce2787e3a69cbef1 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 26 Jun 2026 16:11:52 -0700 Subject: [PATCH 1/2] fix(jobs): stop the reaper from revoking created-but-unstarted jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Jobs can be created without starting them immediately — for example, to pre-configure several jobs and launch them later. Such a job sits in the CREATED state until a user enqueues it. The periodic stale-job reaper (check_stale_jobs) selected jobs by JobState.running_states(), which included CREATED. A pre-configured job older than the staleness cutoff was therefore revoked before it ever ran. A CREATED job has no Celery task and no async (NATS/Redis) resources, so it has nothing to reap or reconcile. enqueue() flips it to PENDING the moment it is dispatched, and a job stuck mid-dispatch is already PENDING (still reaped). Removing CREATED from running_states() also keeps the lost-images reconciler and the async snapshot check from touching jobs that have not started. Closes #1354 Co-Authored-By: Claude --- ami/jobs/models.py | 6 +++++- ami/jobs/tests/test_update_stale_jobs.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/ami/jobs/models.py b/ami/jobs/models.py index e662707f4..63b16a358 100644 --- a/ami/jobs/models.py +++ b/ami/jobs/models.py @@ -79,7 +79,11 @@ class JobState(str, OrderedEnum): @classmethod def running_states(cls): - return [cls.CREATED, cls.PENDING, cls.STARTED, cls.RETRY, cls.CANCELING, cls.UNKNOWN] + # CREATED is excluded: a created-but-not-enqueued job has no Celery task + # and no async resources, so it must never be reaped or reconciled. Jobs + # can be pre-configured and started later, and they wait in CREATED until + # a user enqueues them (which flips them to PENDING). See #1354. + return [cls.PENDING, cls.STARTED, cls.RETRY, cls.CANCELING, cls.UNKNOWN] @classmethod def final_states(cls): diff --git a/ami/jobs/tests/test_update_stale_jobs.py b/ami/jobs/tests/test_update_stale_jobs.py index 0f0a2a215..a64c18e68 100644 --- a/ami/jobs/tests/test_update_stale_jobs.py +++ b/ami/jobs/tests/test_update_stale_jobs.py @@ -121,3 +121,21 @@ def test_skips_recent_and_final_state_jobs(self, mock_cleanup): self.assertEqual(results, []) mock_cleanup.assert_not_called() + + @patch("ami.jobs.tasks.cleanup_async_job_if_needed") + def test_skips_created_but_unstarted_jobs(self, mock_cleanup): + """A job created but never enqueued is left alone regardless of age. + + Jobs can be pre-configured and started later, so a CREATED job has no + Celery task and no async resources to reconcile. The staleness cutoff + (measured against ``updated_at``) does not apply to it — it waits in + CREATED until a user starts it. See #1354. + """ + job = self._create_job(status=JobState.CREATED, minutes_ago=300) + + results = check_stale_jobs() + + self.assertEqual(results, []) + job.refresh_from_db() + self.assertEqual(job.status, JobState.CREATED.value) + mock_cleanup.assert_not_called() From a8ac1a94cd0d1f4c55ae9608dc14707ed145ebbd Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 26 Jun 2026 16:25:14 -0700 Subject: [PATCH 2/2] docs(jobs): tighten JobState comments after dropping CREATED from running_states MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shorten the running_states() note and rewrite the finalizable_states() comment so it no longer describes itself as "running_states() minus ..." — that framing broke once CREATED left running_states() while finalizable_states() still includes it. Describe each set directly. Co-Authored-By: Claude --- ami/jobs/models.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ami/jobs/models.py b/ami/jobs/models.py index 63b16a358..d0e3b0c02 100644 --- a/ami/jobs/models.py +++ b/ami/jobs/models.py @@ -79,10 +79,8 @@ class JobState(str, OrderedEnum): @classmethod def running_states(cls): - # CREATED is excluded: a created-but-not-enqueued job has no Celery task - # and no async resources, so it must never be reaped or reconciled. Jobs - # can be pre-configured and started later, and they wait in CREATED until - # a user enqueues them (which flips them to PENDING). See #1354. + # Dispatched states the reapers/reconcilers select from. CREATED is not + # here: a not-yet-enqueued job has no Celery task to reap. See #1354. return [cls.PENDING, cls.STARTED, cls.RETRY, cls.CANCELING, cls.UNKNOWN] @classmethod @@ -100,8 +98,9 @@ def active_states(cls): @classmethod def finalizable_states(cls): - # running_states() minus CANCELING (don't resurrect a cancel in progress) - # and UNKNOWN (never served; shouldn't auto-finalize). See #1337. + # States a job may auto-finalize from. Excludes CANCELING (don't resurrect + # a cancel in progress) and UNKNOWN (never served; shouldn't auto-finalize). + # See #1337. return [cls.CREATED, cls.PENDING, cls.STARTED, cls.RETRY]