diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 8942f578e..c2c291b78 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -28,6 +28,10 @@ RUN pip install --no-cache-dir -r requirements.txt COPY requirements-webapp.txt ./ RUN pip install --no-cache-dir -r requirements-webapp.txt +# Integration test dependencies (only needed when using AzureIoc) +COPY requirements-integration.txt ./ +RUN pip install --no-cache-dir -r requirements-integration.txt + ENV OPWEN_SESSION_KEY=changeme ENV OPWEN_SETTINGS=/app/docker/client/webapp.env @@ -56,9 +60,13 @@ RUN apt-get update \ # hadolint ignore=DL3010 COPY --from=compiler /app/dist/pkg.tar.gz /app/dist/pkg.tar.gz +COPY --from=compiler /app/requirements-integration.txt /app/requirements-integration.txt +# Install without [opwen_email_server] extra to avoid server dependencies (libcloud 3.2.0) +# Client only needs base package with requirements-webapp.txt (libcloud 3.8.0) # hadolint ignore=DL3013 -RUN pip install --no-cache-dir "/app/dist/pkg.tar.gz[opwen_email_server]" \ +RUN pip install --no-cache-dir "/app/dist/pkg.tar.gz" \ + && pip install --no-cache-dir -r /app/requirements-integration.txt \ && rm -rf /tmp/pip-ephem-wheel-cache* COPY --from=compiler /app/docker/client/run-*.sh /app/docker/client/ diff --git a/opwen_email_server/integration/celery.py b/opwen_email_server/integration/celery.py index b9d2849fc..fb214ea15 100644 --- a/opwen_email_server/integration/celery.py +++ b/opwen_email_server/integration/celery.py @@ -1,4 +1,6 @@ from celery import Celery +from kombu import Exchange +from kombu import Queue from opwen_email_server import config from opwen_email_server.actions import IndexReceivedEmailForMailbox @@ -121,6 +123,29 @@ def _fqn(task): return f'{__name__}.{task.__name__}' +# RabbitMQ 4.0+ deprecated transient non-exclusive queues +# Explicitly define all queues as durable +default_exchange = Exchange('celery', type='direct', durable=True) + +task_queues = ( + Queue(config.REGISTER_CLIENT_QUEUE, + exchange=default_exchange, + routing_key=config.REGISTER_CLIENT_QUEUE, + durable=True), + Queue(config.MAILBOX_RECEIVED_QUEUE, + exchange=default_exchange, + routing_key=config.MAILBOX_RECEIVED_QUEUE, + durable=True), + Queue(config.MAILBOX_SENT_QUEUE, exchange=default_exchange, routing_key=config.MAILBOX_SENT_QUEUE, durable=True), + Queue(config.PROCESS_SERVICE_QUEUE, + exchange=default_exchange, + routing_key=config.PROCESS_SERVICE_QUEUE, + durable=True), + Queue(config.INBOUND_STORE_QUEUE, exchange=default_exchange, routing_key=config.INBOUND_STORE_QUEUE, durable=True), + Queue(config.WRITTEN_STORE_QUEUE, exchange=default_exchange, routing_key=config.WRITTEN_STORE_QUEUE, durable=True), + Queue(config.SEND_QUEUE, exchange=default_exchange, routing_key=config.SEND_QUEUE, durable=True), +) + task_routes = { _fqn(register_client): {'queue': config.REGISTER_CLIENT_QUEUE}, _fqn(index_received_email_for_mailbox): {'queue': config.MAILBOX_RECEIVED_QUEUE}, @@ -131,7 +156,16 @@ def _fqn(task): _fqn(send): {'queue': config.SEND_QUEUE} } -celery.conf.update(task_routes=task_routes) +celery.conf.update( + task_queues=task_queues, + task_routes=task_routes, + # RabbitMQ 4.0+ deprecated transient non-exclusive queues + # Disable remote control to avoid creating non-durable pidbox queues + worker_enable_remote_control=False, + # RabbitMQ 4.0+ deprecated global_qos feature + # Use per-consumer prefetch instead of global QoS + worker_prefetch_multiplier=1, +) if __name__ == '__main__': celery.start() diff --git a/requirements-integration.txt b/requirements-integration.txt new file mode 100644 index 000000000..09f5aebe0 --- /dev/null +++ b/requirements-integration.txt @@ -0,0 +1,15 @@ +# Additional dependencies required for integration testing with AzureIoc +# These are server-side dependencies imported by opwen_email_server.integration.webapp.AzureIoc +# Only needed when LOKOLE_IOC=opwen_email_server.integration.webapp.AzureIoc (integration tests) +# Production client uses opwen_email_client.webapp.ioc.Ioc which doesn't need these + +applicationinsights==0.11.10 # Required by opwen_email_server.utils.log +msgpack==1.0.4 # Required by opwen_email_server.utils.serialization +python-http-client==3.3.7 # Required by sendgrid +sendgrid==6.9.7 # Required by opwen_email_server.integration.celery.send_and_index_email +pyzmail36==1.0.5 # Required by opwen_email_server email parsing +azure-servicebus==7.8.0 # Required by opwen_email_server.services.queue +connexion[swagger-ui]==2.14.0 # Required by opwen_email_server API +jsonschema==4.23.0 # Required by connexion +referencing==0.35.1 # Required by jsonschema +wikipedia==1.4.0 # Required by opwen_email_server.services.index diff --git a/requirements-webapp.txt b/requirements-webapp.txt index bd09e9e0e..b649a1125 100644 --- a/requirements-webapp.txt +++ b/requirements-webapp.txt @@ -1,4 +1,4 @@ -setuptools>=65.5.0,<73.0.0 # Required for pkg_resources in Python 3.12+ (no longer included by default). Upper bound because setuptools 73+ removed pkg_resources +setuptools>=65.5.0,<73.0.0 # Required for pkg_resources in Python 3.12+ (setuptools 73+ removed it) Babel==2.14.0 # 2.14.0+ required for Python 3.13 (removed cgi module dependency) Flask-BabelEx==0.9.4 pytz # Required by Flask-BabelEx (no longer a transitive dep of Babel 2.14+) @@ -16,7 +16,7 @@ WTForms==3.0.1 email-validator==1.2.1 Werkzeug==2.2.1 fasteners==0.17.3 -apache-libcloud==3.8.0 +apache-libcloud==3.8.0 # Client: Python 3.12 support. Overrides base 3.2.0. Azure Storage only. bcrypt==4.0.1 beautifulsoup4==4.11.1 cached-property==1.5.2 diff --git a/requirements.txt b/requirements.txt index d98b8e686..c5e315939 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ Flask==2.2.1 Flask-Cors==3.0.10 Pillow==10.4.0 # 10.1.0+ required for Python 3.13 support setuptools>=65.5.0,<73.0.0 # Required for pkg_resources (gunicorn dependency). setuptools 73+ removed pkg_resources -apache-libcloud==3.8.0 +apache-libcloud==3.2.0 # Server: Python 3.9, stable Cloudflare DNS. Client overrides to 3.8.0 (Python 3.12). applicationinsights==0.11.10 beautifulsoup4==4.11.1 cached-property==1.5.2 diff --git a/tests/opwen_email_server/integration/test_celery.py b/tests/opwen_email_server/integration/test_celery.py index e39f8415c..3d31b9b86 100644 --- a/tests/opwen_email_server/integration/test_celery.py +++ b/tests/opwen_email_server/integration/test_celery.py @@ -25,7 +25,7 @@ def exchange(self) -> Exchange: @cached_property def queue(self) -> Queue: - return Queue(self.queue_name, exchange=self.exchange, routing_key=self.routing_key) + return Queue(self.queue_name, exchange=self.exchange, routing_key=self.routing_key, durable=True) @skipUnless(QUEUE_BROKER, 'no celery broker configured') def test_send_message(self):