From 9b3a8721820eab1db6fa5e43c4e078714e2ff93b Mon Sep 17 00:00:00 2001 From: Karl Anderson Date: Tue, 15 Sep 2026 09:03:10 -0700 Subject: [PATCH 1/7] Update docker image source for minio and minio-init from docker hub to quay.io. --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 703ecea0d..152d9ae42 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -139,7 +139,7 @@ services: - antenna_network minio: - image: minio/minio:RELEASE.2024-11-07T00-52-20Z + image: quay.io/minio/minio:RELEASE.2024-11-07T00-52-20Z command: minio server --console-address ":9001" /data volumes: - "minio_data:/data" @@ -165,7 +165,7 @@ services: - minio minio-init: - image: minio/mc:RELEASE.2025-03-12T17-29-24Z + image: quay.io/minio/mc:RELEASE.2025-03-12T17-29-24Z env_file: - ./.envs/.local/.django depends_on: From 775e1da6ba22c8678430ef31a6dd72dc79eca9e7 Mon Sep 17 00:00:00 2001 From: Karl Anderson Date: Tue, 15 Sep 2026 11:53:39 -0700 Subject: [PATCH 2/7] update docker-compose.ci.yml also --- docker-compose.ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 57f6fbc9f..0bde678d3 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -63,7 +63,7 @@ services: - ./.envs/.ci/.django minio: - image: minio/minio:RELEASE.2024-11-07T00-52-20Z + image: quay.io/minio/minio:RELEASE.2024-11-07T00-52-20Z command: minio server --console-address ":9001" /data volumes: - "minio_ci_data:/data" @@ -76,7 +76,7 @@ services: retries: 5 minio-init: - image: minio/mc:RELEASE.2025-03-12T17-29-24Z + image: quay.io/minio/mc:RELEASE.2025-03-12T17-29-24Z env_file: - ./.envs/.ci/.django depends_on: From 77de55c8eeb9161d7710cc574897ebc09698e0c0 Mon Sep 17 00:00:00 2001 From: Karl Anderson Date: Tue, 15 Sep 2026 15:11:41 -0700 Subject: [PATCH 3/7] Normalize config between the CI and non-CI docker versions --- .envs/.ci/.django | 24 ++++++++++-------------- .envs/.local/.django | 21 +++++++++++---------- docker-compose.ci.yml | 9 +++++---- docker-compose.yml | 2 +- 4 files changed, 27 insertions(+), 29 deletions(-) diff --git a/.envs/.ci/.django b/.envs/.ci/.django index 6577ebece..9d5240299 100644 --- a/.envs/.ci/.django +++ b/.envs/.ci/.django @@ -1,10 +1,14 @@ # General -# ------------------------------------------------------------------------------ -USE_DOCKER=yes DJANGO_SETTINGS_MODULE="config.settings.local" +USE_DOCKER=yes + +# RabbitMQ +CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672/ +CELERY_RESULT_BACKEND=rpc:// # Use RabbitMQ for results backend +RABBITMQ_DEFAULT_USER=rabbituser +RABBITMQ_DEFAULT_PASS=rabbitpass # Redis -# ------------------------------------------------------------------------------ REDIS_URL=redis://redis:6379/0 DJANGO_CSRF_TRUSTED_ORIGINS=http://localhost:3000,http://localhost:4000 @@ -17,16 +21,8 @@ MINIO_STORAGE_USE_HTTPS=False MINIO_TEST_BUCKET=ami-test-ci MINIO_BROWSER_REDIRECT_URL=http://minio:9001 -DEFAULT_PROCESSING_SERVICE_NAME=Test Processing Service -DEFAULT_PROCESSING_SERVICE_ENDPOINT=http://ml_backend:2000 - -# RabbitMQ -# ------------------------------------------------------------------------------ -CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672/ -CELERY_RESULT_BACKEND=rpc:// # Use RabbitMQ for results backend -RABBITMQ_DEFAULT_USER=rabbituser -RABBITMQ_DEFAULT_PASS=rabbitpass - # NATS -# ------------------------------------------------------------------------------ NATS_URL=nats://nats:4222 + +DEFAULT_PROCESSING_SERVICE_NAME=Test Processing Service +DEFAULT_PROCESSING_SERVICE_ENDPOINT=http://ml_backend:2000 diff --git a/.envs/.local/.django b/.envs/.local/.django index 5f65ac07c..364463169 100644 --- a/.envs/.local/.django +++ b/.envs/.local/.django @@ -9,12 +9,17 @@ IPYTHONDIR=/app/.ipython DJANGO_SUPERUSER_EMAIL=antenna@insectai.org DJANGO_SUPERUSER_PASSWORD=localadmin +# RabbitMQ +CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672/ +CELERY_RESULT_BACKEND=rpc:// # Use RabbitMQ for results backend +RABBITMQ_DEFAULT_USER=rabbituser +RABBITMQ_DEFAULT_PASS=rabbitpass +RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=-rabbit consumer_timeout 604800000 + + # Redis REDIS_URL=redis://redis:6379/0 -# NATS -NATS_URL=nats://nats:4222 - # Upload size limit (integer, binary MiB — multiplied by 1024*1024). 100 MiB default # is fine for local dev. Raise only if testing large ML result payloads locally. # DJANGO_DATA_UPLOAD_MAX_MEMORY_MB=100 @@ -28,13 +33,6 @@ CELERY_FLOWER_PASSWORD=BEQgmCtgyrFieKNoGTsux9YIye0I7P5Q7vEgfJD2C4jxmtHDetFaE2jhS # each prefork worker is a separate Python process with its own DB connection. # CELERY_WORKER_CONCURRENCY=4 -# RabbitMQ -CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672/ -CELERY_RESULT_BACKEND=rpc:// # Use RabbitMQ for results backend -RABBITMQ_DEFAULT_USER=rabbituser -RABBITMQ_DEFAULT_PASS=rabbitpass -RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=-rabbit consumer_timeout 604800000 - # Attempting to keep Flower from showing workers as offline # FLOWER_BROKER_API=REDIS_URL FLOWER_PERSISTENT=True @@ -57,6 +55,9 @@ MINIO_STORAGE_USE_HTTPS=False MINIO_TEST_BUCKET=ami-test MINIO_BROWSER_REDIRECT_URL=http://minio:9001 +# NATS +NATS_URL=nats://nats:4222 + # Default processing service (local) DEFAULT_PROCESSING_SERVICE_NAME=Local Processing Service DEFAULT_PROCESSING_SERVICE_ENDPOINT=http://ml_backend:2000 diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 0bde678d3..1d4edc856 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -12,8 +12,6 @@ services: build: context: . dockerfile: ./compose/local/django/Dockerfile - volumes: - - .:/app:z extra_hosts: - "host.docker.internal:host-gateway" depends_on: @@ -21,8 +19,10 @@ services: - redis - minio-init - ml_backend - - rabbitmq - nats + - rabbitmq + volumes: + - .:/app:z env_file: - ./.envs/.ci/.django - ./.envs/.ci/.postgres @@ -39,6 +39,7 @@ services: redis: image: redis:6 + container_name: ami_ci_redis nats: image: nats:2.10-alpine @@ -53,9 +54,9 @@ services: celeryworker: <<: *django + command: /start-celeryworker depends_on: - rabbitmq - command: /start-celeryworker rabbitmq: image: rabbitmq:3-management diff --git a/docker-compose.yml b/docker-compose.yml index 152d9ae42..f91e6d71c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,9 +21,9 @@ services: depends_on: - postgres - redis - - nats - minio-init - ml_backend + - nats - rabbitmq volumes: - .:/app:z From a83b3b9c79a8968869a9869a8fd8faa34f96e787 Mon Sep 17 00:00:00 2001 From: Karl Anderson Date: Tue, 15 Sep 2026 15:11:41 -0700 Subject: [PATCH 4/7] Normalize config between the CI and non-CI docker versions --- .envs/.ci/.django | 24 ++++++++++-------------- .envs/.local/.django | 21 +++++++++++---------- docker-compose.ci.yml | 11 ++++++----- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/.envs/.ci/.django b/.envs/.ci/.django index 6577ebece..9d5240299 100644 --- a/.envs/.ci/.django +++ b/.envs/.ci/.django @@ -1,10 +1,14 @@ # General -# ------------------------------------------------------------------------------ -USE_DOCKER=yes DJANGO_SETTINGS_MODULE="config.settings.local" +USE_DOCKER=yes + +# RabbitMQ +CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672/ +CELERY_RESULT_BACKEND=rpc:// # Use RabbitMQ for results backend +RABBITMQ_DEFAULT_USER=rabbituser +RABBITMQ_DEFAULT_PASS=rabbitpass # Redis -# ------------------------------------------------------------------------------ REDIS_URL=redis://redis:6379/0 DJANGO_CSRF_TRUSTED_ORIGINS=http://localhost:3000,http://localhost:4000 @@ -17,16 +21,8 @@ MINIO_STORAGE_USE_HTTPS=False MINIO_TEST_BUCKET=ami-test-ci MINIO_BROWSER_REDIRECT_URL=http://minio:9001 -DEFAULT_PROCESSING_SERVICE_NAME=Test Processing Service -DEFAULT_PROCESSING_SERVICE_ENDPOINT=http://ml_backend:2000 - -# RabbitMQ -# ------------------------------------------------------------------------------ -CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672/ -CELERY_RESULT_BACKEND=rpc:// # Use RabbitMQ for results backend -RABBITMQ_DEFAULT_USER=rabbituser -RABBITMQ_DEFAULT_PASS=rabbitpass - # NATS -# ------------------------------------------------------------------------------ NATS_URL=nats://nats:4222 + +DEFAULT_PROCESSING_SERVICE_NAME=Test Processing Service +DEFAULT_PROCESSING_SERVICE_ENDPOINT=http://ml_backend:2000 diff --git a/.envs/.local/.django b/.envs/.local/.django index 5f65ac07c..364463169 100644 --- a/.envs/.local/.django +++ b/.envs/.local/.django @@ -9,12 +9,17 @@ IPYTHONDIR=/app/.ipython DJANGO_SUPERUSER_EMAIL=antenna@insectai.org DJANGO_SUPERUSER_PASSWORD=localadmin +# RabbitMQ +CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672/ +CELERY_RESULT_BACKEND=rpc:// # Use RabbitMQ for results backend +RABBITMQ_DEFAULT_USER=rabbituser +RABBITMQ_DEFAULT_PASS=rabbitpass +RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=-rabbit consumer_timeout 604800000 + + # Redis REDIS_URL=redis://redis:6379/0 -# NATS -NATS_URL=nats://nats:4222 - # Upload size limit (integer, binary MiB — multiplied by 1024*1024). 100 MiB default # is fine for local dev. Raise only if testing large ML result payloads locally. # DJANGO_DATA_UPLOAD_MAX_MEMORY_MB=100 @@ -28,13 +33,6 @@ CELERY_FLOWER_PASSWORD=BEQgmCtgyrFieKNoGTsux9YIye0I7P5Q7vEgfJD2C4jxmtHDetFaE2jhS # each prefork worker is a separate Python process with its own DB connection. # CELERY_WORKER_CONCURRENCY=4 -# RabbitMQ -CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672/ -CELERY_RESULT_BACKEND=rpc:// # Use RabbitMQ for results backend -RABBITMQ_DEFAULT_USER=rabbituser -RABBITMQ_DEFAULT_PASS=rabbitpass -RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=-rabbit consumer_timeout 604800000 - # Attempting to keep Flower from showing workers as offline # FLOWER_BROKER_API=REDIS_URL FLOWER_PERSISTENT=True @@ -57,6 +55,9 @@ MINIO_STORAGE_USE_HTTPS=False MINIO_TEST_BUCKET=ami-test MINIO_BROWSER_REDIRECT_URL=http://minio:9001 +# NATS +NATS_URL=nats://nats:4222 + # Default processing service (local) DEFAULT_PROCESSING_SERVICE_NAME=Local Processing Service DEFAULT_PROCESSING_SERVICE_ENDPOINT=http://ml_backend:2000 diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 4caa357f3..f5ed9d86a 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -12,8 +12,6 @@ services: build: context: . dockerfile: ./compose/local/django/Dockerfile - volumes: - - .:/app:z extra_hosts: - "host.docker.internal:host-gateway" depends_on: @@ -26,10 +24,12 @@ services: condition: service_completed_successfully ml_backend: condition: service_started - rabbitmq: - condition: service_started nats: condition: service_started + rabbitmq: + condition: service_started + volumes: + - .:/app:z env_file: - ./.envs/.ci/.django - ./.envs/.ci/.postgres @@ -46,6 +46,7 @@ services: redis: image: redis:6 + container_name: ami_ci_redis nats: image: nats:2.10-alpine @@ -60,9 +61,9 @@ services: celeryworker: <<: *django + command: /start-celeryworker depends_on: - rabbitmq - command: /start-celeryworker rabbitmq: image: rabbitmq:3-management From dd585a380f10cbd6bb447b690778965f0f58f5c1 Mon Sep 17 00:00:00 2001 From: Karl Anderson Date: Wed, 30 Sep 2026 14:36:35 -0700 Subject: [PATCH 5/7] make order match docker-compose.yml, not docker-compose.ci.yml, just to make CI match the default --- docker-compose.ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index f5ed9d86a..42012333c 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -19,13 +19,13 @@ services: condition: service_started redis: condition: service_started + nats: + condition: service_started # Wait for bucket setup to succeed so a failure stops the job here, not in later tests. minio-init: condition: service_completed_successfully ml_backend: condition: service_started - nats: - condition: service_started rabbitmq: condition: service_started volumes: From 8295d91f022bf45b37801c05705238c70210448e Mon Sep 17 00:00:00 2001 From: Karl Anderson Date: Tue, 15 Sep 2026 15:11:41 -0700 Subject: [PATCH 6/7] Normalize config between the CI and non-CI docker versions --- .envs/.ci/.django | 24 ++++++++++-------------- .envs/.local/.django | 21 +++++++++++---------- docker-compose.ci.yml | 11 ++++++----- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/.envs/.ci/.django b/.envs/.ci/.django index 6577ebece..9d5240299 100644 --- a/.envs/.ci/.django +++ b/.envs/.ci/.django @@ -1,10 +1,14 @@ # General -# ------------------------------------------------------------------------------ -USE_DOCKER=yes DJANGO_SETTINGS_MODULE="config.settings.local" +USE_DOCKER=yes + +# RabbitMQ +CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672/ +CELERY_RESULT_BACKEND=rpc:// # Use RabbitMQ for results backend +RABBITMQ_DEFAULT_USER=rabbituser +RABBITMQ_DEFAULT_PASS=rabbitpass # Redis -# ------------------------------------------------------------------------------ REDIS_URL=redis://redis:6379/0 DJANGO_CSRF_TRUSTED_ORIGINS=http://localhost:3000,http://localhost:4000 @@ -17,16 +21,8 @@ MINIO_STORAGE_USE_HTTPS=False MINIO_TEST_BUCKET=ami-test-ci MINIO_BROWSER_REDIRECT_URL=http://minio:9001 -DEFAULT_PROCESSING_SERVICE_NAME=Test Processing Service -DEFAULT_PROCESSING_SERVICE_ENDPOINT=http://ml_backend:2000 - -# RabbitMQ -# ------------------------------------------------------------------------------ -CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672/ -CELERY_RESULT_BACKEND=rpc:// # Use RabbitMQ for results backend -RABBITMQ_DEFAULT_USER=rabbituser -RABBITMQ_DEFAULT_PASS=rabbitpass - # NATS -# ------------------------------------------------------------------------------ NATS_URL=nats://nats:4222 + +DEFAULT_PROCESSING_SERVICE_NAME=Test Processing Service +DEFAULT_PROCESSING_SERVICE_ENDPOINT=http://ml_backend:2000 diff --git a/.envs/.local/.django b/.envs/.local/.django index 5f65ac07c..364463169 100644 --- a/.envs/.local/.django +++ b/.envs/.local/.django @@ -9,12 +9,17 @@ IPYTHONDIR=/app/.ipython DJANGO_SUPERUSER_EMAIL=antenna@insectai.org DJANGO_SUPERUSER_PASSWORD=localadmin +# RabbitMQ +CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672/ +CELERY_RESULT_BACKEND=rpc:// # Use RabbitMQ for results backend +RABBITMQ_DEFAULT_USER=rabbituser +RABBITMQ_DEFAULT_PASS=rabbitpass +RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=-rabbit consumer_timeout 604800000 + + # Redis REDIS_URL=redis://redis:6379/0 -# NATS -NATS_URL=nats://nats:4222 - # Upload size limit (integer, binary MiB — multiplied by 1024*1024). 100 MiB default # is fine for local dev. Raise only if testing large ML result payloads locally. # DJANGO_DATA_UPLOAD_MAX_MEMORY_MB=100 @@ -28,13 +33,6 @@ CELERY_FLOWER_PASSWORD=BEQgmCtgyrFieKNoGTsux9YIye0I7P5Q7vEgfJD2C4jxmtHDetFaE2jhS # each prefork worker is a separate Python process with its own DB connection. # CELERY_WORKER_CONCURRENCY=4 -# RabbitMQ -CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672/ -CELERY_RESULT_BACKEND=rpc:// # Use RabbitMQ for results backend -RABBITMQ_DEFAULT_USER=rabbituser -RABBITMQ_DEFAULT_PASS=rabbitpass -RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=-rabbit consumer_timeout 604800000 - # Attempting to keep Flower from showing workers as offline # FLOWER_BROKER_API=REDIS_URL FLOWER_PERSISTENT=True @@ -57,6 +55,9 @@ MINIO_STORAGE_USE_HTTPS=False MINIO_TEST_BUCKET=ami-test MINIO_BROWSER_REDIRECT_URL=http://minio:9001 +# NATS +NATS_URL=nats://nats:4222 + # Default processing service (local) DEFAULT_PROCESSING_SERVICE_NAME=Local Processing Service DEFAULT_PROCESSING_SERVICE_ENDPOINT=http://ml_backend:2000 diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 4caa357f3..42012333c 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -12,8 +12,6 @@ services: build: context: . dockerfile: ./compose/local/django/Dockerfile - volumes: - - .:/app:z extra_hosts: - "host.docker.internal:host-gateway" depends_on: @@ -21,6 +19,8 @@ services: condition: service_started redis: condition: service_started + nats: + condition: service_started # Wait for bucket setup to succeed so a failure stops the job here, not in later tests. minio-init: condition: service_completed_successfully @@ -28,8 +28,8 @@ services: condition: service_started rabbitmq: condition: service_started - nats: - condition: service_started + volumes: + - .:/app:z env_file: - ./.envs/.ci/.django - ./.envs/.ci/.postgres @@ -46,6 +46,7 @@ services: redis: image: redis:6 + container_name: ami_ci_redis nats: image: nats:2.10-alpine @@ -60,9 +61,9 @@ services: celeryworker: <<: *django + command: /start-celeryworker depends_on: - rabbitmq - command: /start-celeryworker rabbitmq: image: rabbitmq:3-management From 869ed0dbcd5f1a5d2597a9ad0838773b035b4092 Mon Sep 17 00:00:00 2001 From: Karl Anderson Date: Tue, 6 Oct 2026 14:47:16 -0700 Subject: [PATCH 7/7] remove container_name from ci docker compose file to avoid a container being reused when running both ci and local at the same time --- docker-compose.ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 42012333c..e0913da12 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -46,7 +46,6 @@ services: redis: image: redis:6 - container_name: ami_ci_redis nats: image: nats:2.10-alpine